iev 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cca9ca5a2a313fce390cb65045716edd91c671375f3f6b9abd65b57a7dc78c7
-  data.tar.gz: 87726ed4c13880b63483f244683e8c5018a108ef181c8f2b1979a0825998420b
+  metadata.gz: a5909046a19d741d4242dd6823366923efb0607de3ebf89fdcb99465310b6150
+  data.tar.gz: 55812547af56e58d9ea68a1cc7413632558c51a8d3b0d22b48ee61e81afef390
 SHA512:
-  metadata.gz: 2f47e2ef08727216cb287dd520bba49fc89ceca6e4b9c81fff402056b80267cd888c19d462fd68dff413a4f9742ecaa6b8dec90fa17fbcab52fdbb7be3e51ed8
-  data.tar.gz: 6c6f1f7b74a955504625a8fb88a79f302c04571151dad59981644777fec206f06eb612bf5a947f5327e42598ab23c818f26979c73f9c09035bfde9146d69ef60
+  metadata.gz: 6e5cca48906d1f65292cc58887b89564ae293da6cd081e47e70495d11f43c606b6db11d7d99db92210909b4fc3ccfcb54ffdb003191c17134f15c921cbded6c0
+  data.tar.gz: 02f500f0bf2c1da875402a5f26f709d86ee7a4924fd4dd61796e653e260cd52d6c88fc37c9403bf4db479bdeb59664bc2cd599eaa00f40012ca0893a35a96164

data/.github/workflows/rake.yml

@@ -10,6 +10,6 @@ on:
 
 jobs:
   rake:
-    uses: metanorma/metanorma-build-scripts/.github/workflows/generic-rake.yml@main
+    uses: metanorma/ci/.github/workflows/generic-rake.yml@main
     secrets:
       pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/.github/workflows/release.yml

@@ -0,0 +1,25 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: release
+
+on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: |
+          Next release version. Possible values: x.y.z, major, minor, patch (or pre|rc|etc).
+          Also, you can pass 'skip' to skip 'git tag' and do 'gem push' for the current version
+        required: true
+        default: 'skip'
+  repository_dispatch:
+    types: [ do-release ]
+
+jobs:
+  release:
+    uses: metanorma/ci/.github/workflows/rubygems-release.yml@main
+    with:
+      next_version: ${{ github.event.inputs.next_version }}
+    secrets:
+      rubygems-api-key: ${{ secrets.METANORMA_CI_RUBYGEMS_API_KEY }}
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
+

data/.gitignore

@@ -2,6 +2,7 @@
 /.yardoc
 /_yardoc/
 /coverage/
+/db/
 /doc/
 /pkg/
 /spec/reports/
@@ -11,3 +12,6 @@
 
 # rspec failure tracking
 .rspec_status
+.vscode/
+
+concepts/

data/.rubocop.yml

@@ -6,5 +6,3 @@ inherit_from:
 # local repo-specific modifications
 # ...
 
-AllCops:
-  TargetRubyVersion: 2.5

data/README.adoc

@@ -1,4 +1,4 @@
-= Iev
+= IEV
 
 image:https://img.shields.io/gem/v/iev.svg["Gem Version", link="https://rubygems.org/gems/iev"]
 image:https://github.com/metanorma/iev/workflows/rake/badge.svg["Build Status", link="https://github.com/metanorma/iev/actions?query=workflow%3Arake"]
@@ -31,15 +31,15 @@ Or install it yourself as:
 ----
 # Get term
 
-Iev.get("103-01-02", "en")
+IEV.get("103-01-02", "en")
 => "functional"
 
 # If code not found
-Iev.get("111-11-11", "en")
+IEV.get("111-11-11", "en")
 => ""
 
 # If language not found
-Iev.get("103-01-02", "eee")
+IEV.get("103-01-02", "eee")
 => nil
 ----
 

data/exe/iev-glossarist

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+#
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+class Gem::Specification
+  def this; self; end
+end
+
+require "iev"
+require "iev/cli"
+
+IEV::CLI.start(ARGV)

data/iev.gemspec

@@ -4,7 +4,7 @@ require "iev/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "iev"
-  spec.version       = Iev::VERSION
+  spec.version       = IEV::VERSION
   spec.authors       = ["Ribose Inc."]
   spec.email         = ["open.source@ribose.com"]
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "debug"
@@ -27,5 +27,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "simplecov"
 
-  spec.add_dependency "nokogiri", "~> 1"
+  spec.add_dependency "creek", "~> 2.5"
+  spec.add_dependency "glossarist", "~> 2.0.6"
+  spec.add_dependency "unitsml"
+  spec.add_dependency "plurimath"
+  spec.add_dependency "nokogiri", ">= 1.16.5"
+  spec.add_dependency "relaton", "~> 1.18"
+  spec.add_dependency "sequel", "~> 5.40"
+  spec.add_dependency "sqlite3", "~> 1.7.0"
+  spec.add_dependency "thor", "~> 1.0"
+  spec.add_dependency "zeitwerk", "~> 2.4"
 end

data/lib/iev/cli/command.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module CLI
+    class Command < Thor
+      include CommandHelper
+
+      desc "xlsx2yaml FILE", "Converts Excel IEV exports to YAMLs."
+      def xlsx2yaml(file)
+        handle_generic_options(options)
+        db = Sequel.sqlite
+        DbWriter.new(db).import_spreadsheet(file)
+        ds = filter_dataset(db, options)
+        collection = build_collection_from_dataset(ds)
+        save_collection_to_files(collection, options[:output])
+        summary
+      end
+
+      desc "xlsx2db FILE", "Imports Excel to SQLite database."
+      def xlsx2db(file)
+        handle_generic_options(options)
+        # Instantiating an in-memory db and dumping it later is faster than
+        # just working on file db.
+        db = Sequel.sqlite
+        DbWriter.new(db).import_spreadsheet(file)
+        save_db_to_file(db, options[:output])
+        summary
+      end
+
+      desc "db2yaml DB_FILE", "Exports SQLite to IEV YAMLs."
+      def db2yaml(dbfile)
+        handle_generic_options(options)
+        db = Sequel.sqlite(dbfile)
+        ds = filter_dataset(db, options)
+        collection = build_collection_from_dataset(ds)
+        save_collection_to_files(collection, options[:output])
+        summary
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+
+      # Options must be declared at the bottom because Thor must have commands
+      # defined in advance.
+
+      def self.shared_option(name, methods:, **kwargs)
+        [*methods].each { |m| option name, for: m, **kwargs }
+      end
+
+      shared_option :only_concepts,
+        desc: "Only process concepts with IEVREF matching this argument, " +
+          "'%' and '_' wildcards are supported and have meaning as in SQL " +
+          "LIKE operator",
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :only_languages,
+        desc: "Only export these languages, skip concepts which aren't " +
+          "translated to any of them (comma-separated list, language " +
+          "codes must be as in spreadsheet)",
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :output,
+        desc: "Output directory",
+        aliases: :o,
+        default: Dir.pwd,
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :output,
+        desc: "Output file",
+        aliases: :o,
+        default: File.join(Dir.pwd, "concepts.sqlite3"),
+        methods: :xlsx2db
+
+      shared_option :progress,
+        type: :boolean,
+        desc: "Enables or disables progress indicator. By default disabled " +
+          "when 'CI' environment variable is set and enabled otherwise",
+        methods: %i[xlsx2yaml xlsx2db db2yaml]
+
+      shared_option :debug_term_attributes,
+        desc: "Enables debug messages about term attributes recognition",
+        type: :boolean,
+        default: false,
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :debug_sources,
+        desc: "Enables debug messages about authoritative sources recognition",
+        type: :boolean,
+        default: false,
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :debug_relaton,
+        desc: "Enables debug messages about Relaton integration",
+        type: :boolean,
+        default: false,
+        methods: %i[xlsx2yaml db2yaml]
+
+      shared_option :profile,
+        desc: "Generates profiler reports for this program, requires ruby-prof",
+        type: :boolean,
+        default: false,
+        methods: %i[xlsx2yaml xlsx2db db2yaml]
+    end
+  end
+end

data/lib/iev/cli/command_helper.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module CLI
+    module CommandHelper
+      include CLI::UI
+
+      protected
+
+      def save_collection_to_files(collection, output_dir)
+        Profiler.measure("writing-yamls") do
+          info "Writing concepts to files..."
+          path = File.expand_path("./concepts", output_dir)
+          FileUtils.mkdir_p(path)
+          collection.save_to_files(path)
+        end
+      end
+
+      # Note: Implementation examples here:
+      # https://www.rubydoc.info/github/luislavena/sqlite3-ruby/SQLite3/Backup
+      def save_db_to_file(src_db, dbfile)
+        info "Saving database to a file..."
+        src_db.synchronize do |src_conn|
+          dest_conn = SQLite3::Database.new(dbfile)
+          b = SQLite3::Backup.new(dest_conn, "main", src_conn, "main")
+          b.step(-1)
+          b.finish
+        end
+      end
+
+      def summary
+        info "Done!"
+      end
+
+      def collection_file_path(file, output_dir)
+        output_dir.join(Pathname.new(file).basename.sub_ext(".yaml"))
+      end
+
+      # Handles various generic options, e.g. detailed debug switches.
+      # Assigns some global variables accordingly, so these settings are
+      # available throughout the program.
+      def handle_generic_options(options)
+        $IEV_PROFILE = options[:profile]
+        $IEV_PROGRESS = options.fetch(:progress, !ENV["CI"])
+
+        $IEV_DEBUG = options.to_h.
+          select { |k, _| k.to_s.start_with? "debug_" }.
+          transform_keys { |k| k.to_s.sub("debug_", "").to_sym }
+      end
+
+      def filter_dataset(db, options)
+        query = db[:concepts]
+
+        if options[:only_concepts]
+          query = query.where(Sequel.ilike(:ievref, options[:only_concepts]))
+        end
+
+        if options[:only_languages]
+          query = query.where(language: options[:only_languages].split(","))
+        end
+
+        query
+      end
+
+      def build_collection_from_dataset(dataset)
+        Profiler.measure("building-collection") do
+          Glossarist::ManagedConceptCollection.new.tap do |concept_collection|
+            dataset.each do |row|
+              term = TermBuilder.build_from(row)
+              if term
+                concept = concept_collection.fetch_or_initialize(term.id)
+                concept.add_l10n(term)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/iev/cli/ui.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module CLI
+    # @todo
+    #   Make it thread-safe.  Currently, calling UI methods from different
+    #   threads may result with mangled output.  At first glance it seems like
+    #   something is wrong with carriage returns, but more research is needed.
+    module UI
+      module_function
+
+      def debug(*args)
+        Helper.cli_out(:debug, *args)
+      end
+
+      def warn(*args)
+        Helper.cli_out(:warn, *args)
+      end
+
+      # Prints progress message which will be replaced on next call.
+      def progress(message)
+        return unless $IEV_PROGRESS
+
+        print "#{Helper.clear_progress}#{message} "
+      end
+
+      # Prints generic message.
+      def info(message)
+        print "#{Helper.clear_progress}#{message}\n"
+      end
+
+      # Sets an UI tag which will be prepended to messages printed with
+      # #debug and #warn.
+      def set_ui_tag(str)
+        Thread.current[:iev_ui_tag] = str
+      end
+
+      module Helper
+        module_function
+
+        def clear_progress
+          $IEV_PROGRESS ? "\r#{" " * 40}\r" : ""
+        end
+
+        def cli_out(level, *args)
+          topic = Symbol === args[0] ? args.shift : nil
+          message = args.map(&:to_s).join(" ").chomp
+          ui_tag = Thread.current[:iev_ui_tag]
+
+          return unless should_out?(level, topic)
+
+          print [
+            clear_progress,
+            ui_tag,
+            ui_tag && ": ",
+            message,
+            "\n",
+          ].join
+        end
+
+        def should_out?(level, topic)
+          topic.nil? || level == :warn || $IEV_DEBUG[topic]
+        end
+      end
+    end
+  end
+end

data/lib/iev/cli.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module CLI
+    def self.start(arguments)
+      Signal.trap("INT") do
+        UI.info "Signal SIGINT received, quitting!"
+        Kernel.exit(1)
+      end
+
+      Signal.trap("TERM") do
+        UI.info "Signal SIGTERM received, quitting!"
+        Kernel.exit(1)
+      end
+
+      IEV::CLI::Command.start(arguments)
+    end
+  end
+end

data/lib/iev/converter/mathml_to_asciimath.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+module IEV
+  module Converter
+    class MathmlToAsciimath
+      using DataConversions
+
+      def self.convert(input)
+        new.convert(input)
+      end
+
+      def convert(input)
+        mathml_to_asciimath(input)
+      end
+
+      private
+
+      def mathml_to_asciimath(input)
+        # If given string does not include '<' (for elements) nor '&'
+        # (for entities), then it's certain that it doesn't contain
+        # any MathML or HTML formula.
+        return input unless input&.match?(/<|&/)
+
+        unless input.include?("<math>")
+          return html_to_asciimath(input)
+        end
+
+        # puts "GOING TO MATHML MATH"
+        # puts input
+        to_asciimath = Nokogiri::HTML.fragment(input, "UTF-8")
+        # to_asciimath.remove_namespaces!
+
+        to_asciimath.css("math").each do |math_element|
+          asciimath = Plurimath::Math.parse(
+            text_to_asciimath(math_element.to_xml), :mathml
+          ).to_asciimath.strip
+          # puts"ASCIIMATH!!  #{asciimath}"
+
+          if asciimath.empty?
+            math_element.remove
+          else
+            math_element.replace "stem:[#{asciimath}]"
+          end
+        end
+
+        html_to_asciimath(
+          to_asciimath.children.to_s,
+        )
+      end
+
+      def html_to_asciimath(input)
+        return input if input.nil? || input.empty?
+
+        to_asciimath = Nokogiri::HTML.fragment(input, "UTF-8")
+
+        to_asciimath.css("i").each do |math_element|
+          # puts "HTML MATH!!  #{math_element.to_xml}"
+          # puts "HTML MATH!!  #{math_element.text}"
+          decoded = text_to_asciimath(math_element.text)
+          case decoded.length
+          when 1..12
+            # puts "(#{math_element.text} to => #{decoded})"
+            math_element.replace "stem:[#{decoded}]"
+          when 0
+            math_element.remove
+          else
+            math_element.replace "_#{decoded}_"
+          end
+        end
+
+        to_asciimath.css("sub").each do |math_element|
+          case math_element.text.length
+          when 0
+            math_element.remove
+          else
+            math_element.replace "~#{text_to_asciimath(math_element.text)}~"
+          end
+        end
+
+        to_asciimath.css("sup").each do |math_element|
+          case math_element.text.length
+          when 0
+            math_element.remove
+          else
+            math_element.replace "^#{text_to_asciimath(math_element.text)}^"
+          end
+        end
+
+        to_asciimath.css("ol").each do |element|
+          element.css("li").each do |li|
+            li.replace ". #{li.text}"
+          end
+        end
+
+        to_asciimath.css("ul").each do |element|
+          element.css("li").each do |li|
+            li.replace "* #{li.text}"
+          end
+        end
+
+        # Replace sans-serif font with monospace
+        to_asciimath.css('font[style*="sans-serif"]').each do |x|
+          x.replace "`#{x.text}`"
+        end
+
+        html_entities_to_stem(
+          to_asciimath
+            .children.to_s
+            .gsub(/\]stem:\[/, "")
+            .gsub(/<\/?[uo]l>/, ""),
+        )
+      end
+
+      def text_to_asciimath(text)
+        html_entities_to_asciimath(text.decode_html)
+      end
+
+      def html_entities_to_asciimath(input)
+        input.gsub("&alpha;", "alpha").
+          gsub("&beta;", "beta").
+          gsub("&gamma;", "gamma").
+          gsub("&Gamma;", "Gamma").
+          gsub("&delta;", "delta").
+          gsub("&Delta;", "Delta").
+          gsub("&epsilon;", "epsilon").
+          gsub("&varepsilon;", "varepsilon").
+          gsub("&zeta;", "zeta").
+          gsub("&eta;", "eta").
+          gsub("&theta;", "theta").
+          gsub("&Theta;", "Theta").
+          gsub("&vartheta;", "vartheta").
+          gsub("&iota;", "iota").
+          gsub("&kappa;", "kappa").
+          gsub("&lambda;", "lambda").
+          gsub("&Lambda;", "Lambda").
+          gsub("&mu;", "mu").
+          gsub("&nu;", "nu").
+          gsub("&xi;", "xi").
+          gsub("&Xi;", "Xi").
+          gsub("&pi;", "pi").
+          gsub("&Pi;", "Pi").
+          gsub("&rho;", "rho").
+          gsub("&beta;", "beta").
+          gsub("&sigma;", "sigma").
+          gsub("&Sigma;", "Sigma").
+          gsub("&tau;", "tau").
+          gsub("&upsilon;", "upsilon").
+          gsub("&phi;", "phi").
+          gsub("&Phi;", "Phi").
+          gsub("&varphi;", "varphi").
+          gsub("&chi;", "chi").
+          gsub("&psi;", "psi").
+          gsub("&Psi;", "Psi").
+          gsub("&omega;", "omega")
+      end
+
+      def html_entities_to_stem(input)
+        input.gsub("&alpha;", "stem:[alpha]").
+          gsub("&beta;", "stem:[beta]").
+          gsub("&gamma;", "stem:[gamma]").
+          gsub("&Gamma;", "stem:[Gamma]").
+          gsub("&delta;", "stem:[delta]").
+          gsub("&Delta;", "stem:[Delta]").
+          gsub("&epsilon;", "stem:[epsilon]").
+          gsub("&varepsilon;", "stem:[varepsilon]").
+          gsub("&zeta;", "stem:[zeta]").
+          gsub("&eta;", "stem:[eta]").
+          gsub("&theta;", "stem:[theta]").
+          gsub("&Theta;", "stem:[Theta]").
+          gsub("&vartheta;", "stem:[vartheta]").
+          gsub("&iota;", "stem:[iota]").
+          gsub("&kappa;", "stem:[kappa]").
+          gsub("&lambda;", "stem:[lambda]").
+          gsub("&Lambda;", "stem:[Lambda]").
+          gsub("&mu;", "stem:[mu]").
+          gsub("&nu;", "stem:[nu]").
+          gsub("&xi;", "stem:[xi]").
+          gsub("&Xi;", "stem:[Xi]").
+          gsub("&pi;", "stem:[pi]").
+          gsub("&Pi;", "stem:[Pi]").
+          gsub("&rho;", "stem:[rho]").
+          gsub("&beta;", "stem:[beta]").
+          gsub("&sigma;", "stem:[sigma]").
+          gsub("&Sigma;", "stem:[Sigma]").
+          gsub("&tau;", "stem:[tau]").
+          gsub("&upsilon;", "stem:[upsilon]").
+          gsub("&phi;", "stem:[phi]").
+          gsub("&Phi;", "stem:[Phi]").
+          gsub("&varphi;", "stem:[varphi]").
+          gsub("&chi;", "stem:[chi]").
+          gsub("&psi;", "stem:[psi]").
+          gsub("&Psi;", "stem:[Psi]").
+          gsub("&omega;", "stem:[omega]")
+      end
+    end
+  end
+end

data/lib/iev/converter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module IEV
+  module Converter
+    def self.mathml_to_asciimath(input)
+      IEV::Converter::MathmlToAsciimath.convert(input)
+    end
+  end
+end

data/lib/iev/data_conversions.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# (c) Copyright 2020 Ribose Inc.
+#
+
+module IEV
+  module DataConversions
+    refine String do
+      def decode_html!
+        replace(decode_html)
+        nil
+      end
+
+      def decode_html
+        HTMLEntities.new(:expanded).decode(self)
+      end
+
+      # Normalize various encoding anomalies like `\uFEFF` in strings
+      def sanitize!
+        unicode_normalize!
+        gsub!("\uFEFF", "")
+        gsub!("\u2011", "-")
+        gsub!("\u00a0", " ")
+        gsub!(/[\u2000-\u2006]/, " ")
+        strip!
+        nil
+      end
+
+      # @see sanitize!
+      def sanitize
+        dup.tap(&:sanitize!)
+      end
+
+      def to_three_char_code
+        IEV::Iso639Code.three_char_code(self).first
+      end
+    end
+  end
+end