stepmod-utils 0.2.0 → 0.2.7

data/lib/stepmod/utils/html_to_asciimath.rb

@@ -0,0 +1,157 @@
+module Stepmod
+  module Utils
+    class HtmlToAsciimath
+      def call(input)
+        return input if input.nil? || input.empty?
+
+        to_asciimath = Nokogiri::HTML.fragment(input, "UTF-8")
+
+        to_asciimath.css('i').each do |math_element|
+          # puts "HTML MATH!!  #{math_element.to_xml}"
+          # puts "HTML MATH!!  #{math_element.text}"
+          decoded = text_to_asciimath(math_element.text)
+          case decoded.length
+          when 1..12
+            # puts "(#{math_element.text} to => #{decoded})"
+            math_element.replace "stem:[#{decoded}]"
+          when 0
+            math_element.remove
+          else
+            math_element.replace "_#{decoded}_"
+          end
+        end
+
+        to_asciimath.css('sub').each do |math_element|
+          case math_element.text.length
+          when 0
+            math_element.remove
+          else
+            math_element.replace "~#{text_to_asciimath(math_element.text)}~"
+          end
+        end
+
+        to_asciimath.css('sup').each do |math_element|
+          case math_element.text.length
+          when 0
+            math_element.remove
+          else
+            math_element.replace "^#{text_to_asciimath(math_element.text)}^"
+          end
+        end
+
+        to_asciimath.css('ol').each do |element|
+          element.css('li').each do |li|
+            li.replace ". #{li.text}"
+          end
+        end
+
+        to_asciimath.css('ul').each do |element|
+          element.css('li').each do |li|
+            li.replace "* #{li.text}"
+          end
+        end
+
+        # Replace sans-serif font with monospace
+        to_asciimath.css('font[style*="sans-serif"]').each do |x|
+          x.replace "`#{x.text}`"
+        end
+
+        html_entities_to_stem(
+          to_asciimath.children.to_s.gsub(/\]stem:\[/, '').gsub(/<\/?[uo]l>/, '')
+        )
+      end
+
+      def text_to_asciimath(text)
+        html_entities_to_asciimath(text.decode_html)
+      end
+
+      def html_entities_to_asciimath(x)
+        x.gsub("&alpha;", "alpha").
+          gsub("&beta;", "beta").
+          gsub("&gamma;", "gamma").
+          gsub("&Gamma;", "Gamma").
+          gsub("&delta;", "delta").
+          gsub("&Delta;", "Delta").
+          gsub("&epsilon;", "epsilon").
+          gsub("&varepsilon;", "varepsilon").
+          gsub("&zeta;", "zeta").
+          gsub("&eta;", "eta").
+          gsub("&theta;", "theta").
+          gsub("&Theta;", "Theta").
+          gsub("&vartheta;", "vartheta").
+          gsub("&iota;", "iota").
+          gsub("&kappa;", "kappa").
+          gsub("&lambda;", "lambda").
+          gsub("&Lambda;", "Lambda").
+          gsub("&mu;", "mu").
+          gsub("&nu;", "nu").
+          gsub("&xi;", "xi").
+          gsub("&Xi;", "Xi").
+          gsub("&pi;", "pi").
+          gsub("&Pi;", "Pi").
+          gsub("&rho;", "rho").
+          gsub("&beta;", "beta").
+          gsub("&sigma;", "sigma").
+          gsub("&Sigma;", "Sigma").
+          gsub("&tau;", "tau").
+          gsub("&upsilon;", "upsilon").
+          gsub("&phi;", "phi").
+          gsub("&Phi;", "Phi").
+          gsub("&varphi;", "varphi").
+          gsub("&chi;", "chi").
+          gsub("&psi;", "psi").
+          gsub("&Psi;", "Psi").
+          gsub("&omega;", "omega")
+          gsub("&#967", "χ").
+          gsub("&#215", "×").
+          gsub("&#931", "Σ").
+          gsub("&#961", "ρ").
+          gsub("&#963", "σ").
+          gsub("&#955", "λ").
+          gsub("&#964", "τ").
+          gsub("&#8706", "∂").
+          gsub("&#8804", "≤").
+          gsub("&#8805", "≥")
+      end
+
+      def html_entities_to_stem(x)
+        x.gsub("&alpha;", "stem:[alpha]").
+          gsub("&beta;", "stem:[beta]").
+          gsub("&gamma;", "stem:[gamma]").
+          gsub("&Gamma;", "stem:[Gamma]").
+          gsub("&delta;", "stem:[delta]").
+          gsub("&Delta;", "stem:[Delta]").
+          gsub("&epsilon;", "stem:[epsilon]").
+          gsub("&varepsilon;", "stem:[varepsilon]").
+          gsub("&zeta;", "stem:[zeta]").
+          gsub("&eta;", "stem:[eta]").
+          gsub("&theta;", "stem:[theta]").
+          gsub("&Theta;", "stem:[Theta]").
+          gsub("&vartheta;", "stem:[vartheta]").
+          gsub("&iota;", "stem:[iota]").
+          gsub("&kappa;", "stem:[kappa]").
+          gsub("&lambda;", "stem:[lambda]").
+          gsub("&Lambda;", "stem:[Lambda]").
+          gsub("&mu;", "stem:[mu]").
+          gsub("&nu;", "stem:[nu]").
+          gsub("&xi;", "stem:[xi]").
+          gsub("&Xi;", "stem:[Xi]").
+          gsub("&pi;", "stem:[pi]").
+          gsub("&Pi;", "stem:[Pi]").
+          gsub("&rho;", "stem:[rho]").
+          gsub("&beta;", "stem:[beta]").
+          gsub("&sigma;", "stem:[sigma]").
+          gsub("&Sigma;", "stem:[Sigma]").
+          gsub("&tau;", "stem:[tau]").
+          gsub("&upsilon;", "stem:[upsilon]").
+          gsub("&phi;", "stem:[phi]").
+          gsub("&Phi;", "stem:[Phi]").
+          gsub("&varphi;", "stem:[varphi]").
+          gsub("&chi;", "stem:[chi]").
+          gsub("&psi;", "stem:[psi]").
+          gsub("&Psi;", "stem:[Psi]").
+          gsub("&omega;", "stem:[omega]")
+      end
+    end
+  end
+end

data/lib/stepmod/utils/smrl_description_converter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'reverse_adoc'
+require 'stepmod/utils/converters/a'
+require 'stepmod/utils/converters/blockquote'
+require 'stepmod/utils/converters/br'
+require 'stepmod/utils/converters/bypass'
+require 'stepmod/utils/converters/code'
+require 'stepmod/utils/converters/drop'
+require 'stepmod/utils/converters/em_express_description'
+require 'stepmod/utils/converters/example'
+require 'stepmod/utils/converters/express_ref_express_description'
+require 'stepmod/utils/converters/ext_description'
+require 'stepmod/utils/converters/ext_descriptions'
+require 'stepmod/utils/converters/head'
+require 'stepmod/utils/converters/hr'
+require 'stepmod/utils/converters/ignore'
+require 'stepmod/utils/converters/note'
+require 'stepmod/utils/converters/p'
+require 'stepmod/utils/converters/pass_through'
+require 'stepmod/utils/converters/q'
+require 'stepmod/utils/converters/strong'
+require 'stepmod/utils/converters/sub'
+require 'stepmod/utils/converters/sup'
+require 'stepmod/utils/converters/text'
+require 'stepmod/utils/cleaner'
+
+module Stepmod
+  module Utils
+    class SmrlDescriptionConverter
+      def self.convert(input, options = {})
+        root = if input.is_a?(String)
+                  then Nokogiri::XML(input).root
+                elsif input.is_a?(Nokogiri::XML::Document)
+                  then input.root
+                elsif input.is_a?(Nokogiri::XML::Node)
+                  then input
+                end
+
+        root || (return '')
+
+        ReverseAdoc.config.with(options) do
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          Stepmod::Utils::Cleaner.new.tidy(result)
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'reverse_adoc'
+require 'reverse_adoc/converters/bypass'
+require 'reverse_adoc/converters/pass_through'
+require 'stepmod/utils/converters/a'
+require 'stepmod/utils/converters/blockquote'
+require 'stepmod/utils/converters/br'
+require 'stepmod/utils/converters/bypass'
+require 'stepmod/utils/converters/code'
+require 'stepmod/utils/converters/comment'
+require 'stepmod/utils/converters/dd'
+require 'stepmod/utils/converters/dl'
+require 'stepmod/utils/converters/dt'
+require 'stepmod/utils/converters/drop'
+require 'stepmod/utils/converters/example'
+require 'stepmod/utils/converters/express_g'
+require 'stepmod/utils/converters/fund_cons'
+require 'stepmod/utils/converters/eqn'
+require 'stepmod/utils/converters/head'
+require 'stepmod/utils/converters/hr'
+require 'stepmod/utils/converters/ignore'
+require 'stepmod/utils/converters/introduction'
+require 'stepmod/utils/converters/note'
+require 'stepmod/utils/converters/ol'
+require 'stepmod/utils/converters/p'
+require 'stepmod/utils/converters/pass_through'
+require 'stepmod/utils/converters/q'
+require 'stepmod/utils/converters/resource'
+require 'stepmod/utils/converters/schema_diag'
+require 'stepmod/utils/converters/schema'
+require 'stepmod/utils/converters/strong'
+require 'stepmod/utils/converters/sub'
+require 'stepmod/utils/converters/sup'
+require 'stepmod/utils/converters/table'
+require 'stepmod/utils/converters/text'
+require 'stepmod/utils/cleaner'
+
+require 'reverse_adoc/converters/figure'
+require 'reverse_adoc/converters/img'
+require 'reverse_adoc/converters/li'
+require 'reverse_adoc/converters/tr'
+require 'reverse_adoc/converters/td'
+require 'reverse_adoc/converters/th'
+
+module Stepmod
+  module Utils
+    class SmrlResourceConverter
+      def self.convert(input, options = {})
+        root = if input.is_a?(String)
+                  then Nokogiri::XML(input).root
+                elsif input.is_a?(Nokogiri::XML::Document)
+                  then input.root
+                elsif input.is_a?(Nokogiri::XML::Node)
+                  then input
+                end
+
+        root || (return '')
+
+        ReverseAdoc.config.with(options) do
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          Stepmod::Utils::Cleaner.new.tidy(result)
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -0,0 +1,78 @@
+require 'json'
+require 'stepmod/utils/smrl_description_converter'
+require 'stepmod/utils/smrl_resource_converter'
+
+module Stepmod
+  module Utils
+    class StepmodFileAnnotator
+      attr_reader :express_file, :resource_docs_cache_file, :stepmod_dir
+
+      # @param express_file [String] path to the exp file needed to annotate
+      # @param resource_docs_cache_file [String] output of ./stepmod-build-resource-docs-cache
+      def initialize(express_file:, resource_docs_cache_file:, stepmod_dir: nil)
+        @express_file = express_file
+        @resource_docs_cache_file = resource_docs_cache_file
+        @stepmod_dir = stepmod_dir || Dir.pwd
+      end
+
+      def call
+        match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
+        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : 'descriptions.xml'
+        descriptions_file = File.join(File.dirname(express_file), descriptions_base)
+        output_express = File.read(express_file)
+        resource_docs_cache = JSON.parse(File.read(resource_docs_cache_file))
+
+        if File.exists?(descriptions_file)
+          descriptions = Nokogiri::XML(File.read(descriptions_file)).root
+          added_resource_descriptions = {}
+          descriptions.xpath('ext_description').each do |description|
+            # Add base resource from linked path if exists, eg "language_schema.language.wr:WR1" -> "language_schema"
+            base_linked = description['linkend'].to_s.split('.').first
+            if added_resource_descriptions[base_linked].nil?
+              base_reource_doc_dir = resource_docs_cache[description['linkend'].to_s.split('.').first]
+              if base_reource_doc_dir
+                output_express << convert_from_resource_file(base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file)
+              end
+              added_resource_descriptions[base_linked] = true
+            end
+            resource_docs_dir = resource_docs_cache[description['linkend']]
+            # Convert content description
+            # when a schema description is available from resource.xml and also descriptions.xml, the description from resource.xml is only used.
+            # https://github.com/metanorma/annotated-express/issues/32#issuecomment-792609078
+            if description.text.strip.length.positive? && resource_docs_dir.nil?
+              output_express << convert_from_description_text(descriptions_file, description)
+            end
+            # Add converted description from exact linked path
+            if resource_docs_dir && added_resource_descriptions[description['linkend']].nil?
+              output_express << convert_from_resource_file(resource_docs_dir, stepmod_dir, description['linkend'], descriptions_file)
+              added_resource_descriptions[description['linkend']] = true
+            end
+          end
+        end
+
+        output_express
+      end
+
+      private
+
+      def convert_from_description_text(descriptions_file, description)
+        Dir.chdir(File.dirname(descriptions_file)) do
+          wrapper = "<ext_descriptions>#{description.to_s}</ext_descriptions>"
+          "\n" + Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)
+        end
+      end
+
+      def convert_from_resource_file(resource_docs_dir, stepmod_dir, linked, descriptions_file)
+        resource_docs_file = File.join(stepmod_dir, 'data/resource_docs', resource_docs_dir, 'resource.xml')
+        puts(resource_docs_file)
+        resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
+        schema = resource_docs.xpath("schema[@name='#{linked}']")
+
+        Dir.chdir(File.dirname(descriptions_file)) do
+          wrapper = "<resource>#{schema.to_s}</resource>"
+          "\n" + Stepmod::Utils::SmrlResourceConverter.convert(wrapper)
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.2.0"
+    VERSION = "0.2.7"
   end
 end

data/migrating_from_cvs.adoc

@@ -0,0 +1,190 @@
+= STEPmod CVS to Git import
+
+== Purpose
+
+The `cvs/` submodule is a CSV import of STEPmod files from the CVS server hosted at Boost Conseil.
+
+The goal is to import the CSV-managed files in a way useable via Git, with full history information.
+
+This guide is used as reference for the usage of `cvs-fast-import`:
+https://oitofelix.github.io/article-savannah-cvs-to-git-migration/
+
+
+== Strategy to import
+
+Cloning a remote CVS repository while importing is super slow,
+especially for a large repository like STEPmod.
+
+We have tested and settled on these steps:
+
+. Maintain a local `rsync` copy of the CVS repository.
+. Resolve all names from the CVS repository (CVS only stores UNIX usernames, in Git are names and emails) using `cvs-fast-import`.
+. Run `cvs-fast-import` to import the CVS repository into the `iso-10303-stepmod-cvs` Git repo.
+
+
+== Creating the `rsync` clone of the CVS repository
+
+[source,sh]
+----
+rsync -avrPz -e ssh ronald@cvs.boost-lab.net:/stepmod/ stepmod-rsync/
+----
+
+
+== Install cvs-fast-export
+
+`cvs-fast-export` only works on Linux.
+Run it on Ubuntu with the `rsync`'ed CVS directory.
+
+Install:
+[source,sh]
+----
+$ apt-get -y install cvs-fast-export
+----
+
+== Getting all users for email mapping
+
+Find all authors in the CVS repository using `cvs-fast-export -a`.
+It is much faster than using the equivalent CVS command to list all authors.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -a
+----
+
+
+== Create fast-import file for Git
+
+After all authors are mapped, run `cvs-fast-export` to create
+the import file.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -A author-map.txt > fast-import-file
+----
+
+== Perform the Git fast-import
+
+Once the fast-import file is created, we can perform the Git import.
+
+[source,sh]
+----
+$ cd iso-10303-stepmod-cvs
+$ git fast-import < ../fast-import-file
+----
+
+
+== Upload the new Git repo
+
+[source,sh]
+----
+git push --all && git push --tags
+----
+
+
+== DEPRECATED steps using `git cvsimport` (do not use, it won't work)
+
+=== General
+
+https://stackoverflow.com/questions/11362676/how-to-import-and-keep-updated-a-cvs-repository-in-git[This StackOverflow post]
+describes steps for using `git cvsimport`.
+
+Originally the `git cvsimport` tool was chosen since it is part of
+`git`. However, it utilizes a deprecated/unmaintained tool called
+`cvsps`. The latest `cvsps` is version 3, but only version 2 is
+compatible with `git cvsimport`.
+
+In addition, the `cvsps` tool is maintained by the maintainer
+of `cvs-fast-import`, and is no longer updated.
+Eventually `cvs-fast-import` is used instead.
+
+WARNING: This command completely fails
+on this repository because it is too large and complex.
+
+
+=== Setup
+
+On macOS, run the following commands to setup for running the import. The `git` executable must be installed.
+
+Install `cvsps` version 2.
+
+NOTE: The steps from the StackOverflow of installing `cvsps` no longer work.
+
+[source,sh]
+----
+$ brew tap Frizlab/Perso
+# ==> Tapping frizlab/perso
+# Cloning into '/usr/local/Homebrew/Library/Taps/frizlab/homebrew-perso'...
+# remote: Enumerating objects: 123, done.
+# remote: Total 123 (delta 0), reused 0 (delta 0), pack-reused 123
+# Receiving objects: 100% (123/123), 19.08 KiB | 91.00 KiB/s, done.
+# Resolving deltas: 100% (43/43), done.
+# Tapped 1 cask and 10 formulae (38 files, 60.5KB).
+
+$ brew install cvsps@2
+# ==> Installing cvsps@2 from frizlab/perso
+# Warning: A newer Command Line Tools release is available.
+# Update them from Software Update in System Preferences or
+# https://developer.apple.com/download/more/.
+# ==> Downloading https://deb.debian.org/debian/pool/main/c/cvsps/cvsps_2.1.orig.tar.gz
+######################################################################## 100.0%
+# ==> make all
+# ==> make install prefix=/usr/local/Cellar/cvsps@2/2.1
+# 🍺  /usr/local/Cellar/cvsps@2/2.1: 7 files, 124.6KB, built in 6 seconds
+----
+
+Verify it is installed:
+[source,sh]
+----
+$ cvsps -v
+# Can't open CVS/Root
+# cannot determine CVSROOT
+----
+
+////
+== Checkout the CVS repository
+
+First set the `CVSROOT` and `CVS_RSH` variables.
+
+[source,sh]
+----
+export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+export CVS_RSH=ssh
+cvs checkout stepmod
+# => stepmod/ is created in $PWD
+----
+////
+
+
+=== Run the import
+
+Run the import from CVS to Git.
+
+Go to the directory that will carry the target Git repository.
+
+Run `git cvsimport`.
+
+[source,sh]
+----
+# Importing from remote
+$ export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+$ export CVS_RSH=ssh
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+
+# Importing from local rsync'ed copy
+$ export CVSROOT=$(pwd)/stepmod-rsync
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+----
+
+WARNING: TLDR. Technically this should work, but I ran into a `cvsps cannot allocate memory` error with 64GB of RAM, and not even completing the clone after 24 hours. So I gave up and switched to local. And local still takes a long time with tons of error messages. `cvs-fast-import` only takes 5-10 minutes to import.
+
+
+=== Updating the CVS import
+
+Run `git cvsimport` to synchronize the Git repo using updated data from CVS.
+
+[source,sh]
+----
+$ git cvsimport
+----
+
+