stepmod-utils 0.1.6 → 0.2.4

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'reverse_adoc'
+require 'reverse_adoc/converters/bypass'
+require 'reverse_adoc/converters/pass_through'
+require 'stepmod/utils/converters/a'
+require 'stepmod/utils/converters/blockquote'
+require 'stepmod/utils/converters/br'
+require 'stepmod/utils/converters/bypass'
+require 'stepmod/utils/converters/code'
+require 'stepmod/utils/converters/comment'
+require 'stepmod/utils/converters/dd'
+require 'stepmod/utils/converters/dl'
+require 'stepmod/utils/converters/dt'
+require 'stepmod/utils/converters/drop'
+require 'stepmod/utils/converters/example'
+require 'stepmod/utils/converters/express_g'
+require 'stepmod/utils/converters/fund_cons'
+require 'stepmod/utils/converters/eqn'
+require 'stepmod/utils/converters/head'
+require 'stepmod/utils/converters/hr'
+require 'stepmod/utils/converters/ignore'
+require 'stepmod/utils/converters/introduction'
+require 'stepmod/utils/converters/note'
+require 'stepmod/utils/converters/ol'
+require 'stepmod/utils/converters/p'
+require 'stepmod/utils/converters/pass_through'
+require 'stepmod/utils/converters/q'
+require 'stepmod/utils/converters/resource'
+require 'stepmod/utils/converters/schema_diag'
+require 'stepmod/utils/converters/schema'
+require 'stepmod/utils/converters/strong'
+require 'stepmod/utils/converters/sub'
+require 'stepmod/utils/converters/sup'
+require 'stepmod/utils/converters/text'
+require 'stepmod/utils/cleaner'
+
+require 'reverse_adoc/converters/figure'
+require 'reverse_adoc/converters/img'
+require 'reverse_adoc/converters/li'
+require 'reverse_adoc/converters/table'
+require 'reverse_adoc/converters/tr'
+require 'reverse_adoc/converters/td'
+require 'reverse_adoc/converters/th'
+
+module Stepmod
+  module Utils
+    class SmrlResourceConverter
+      def self.convert(input, options = {})
+        root = if input.is_a?(String)
+                  then Nokogiri::XML(input).root
+                elsif input.is_a?(Nokogiri::XML::Document)
+                  then input.root
+                elsif input.is_a?(Nokogiri::XML::Node)
+                  then input
+                end
+
+        root || (return '')
+
+        ReverseAdoc.config.with(options) do
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          Stepmod::Utils::Cleaner.new.tidy(result)
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -0,0 +1,54 @@
+require 'json'
+require 'stepmod/utils/smrl_description_converter'
+require 'stepmod/utils/smrl_resource_converter'
+
+module Stepmod
+  module Utils
+    class StepmodFileAnnotator
+      attr_reader :express_file, :resource_docs_cache_file, :stepmod_dir
+
+      # @param express_file [String] path to the exp file needed to annotate
+      # @param resource_docs_cache_file [String] output of ./stepmod-build-resource-docs-cache
+      def initialize(express_file:, resource_docs_cache_file:, stepmod_dir: nil)
+        @express_file = express_file
+        @resource_docs_cache_file = resource_docs_cache_file
+        @stepmod_dir = stepmod_dir || Dir.pwd
+      end
+
+      def call
+        match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
+        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : 'descriptions.xml'
+        descriptions_file = File.join(File.dirname(express_file), descriptions_base)
+        output_express = File.read(express_file)
+        resource_docs_cache = JSON.parse(File.read(resource_docs_cache_file))
+
+        if File.exists?(descriptions_file)
+          descriptions = Nokogiri::XML(File.read(descriptions_file)).root
+          descriptions.xpath('ext_description').each do |description|
+            unless description.text.strip.empty? then
+              Dir.chdir(File.dirname(descriptions_file)) do
+                wrapper = "<ext_descriptions>#{description.to_s}</ext_descriptions>"
+                output_express << "\n" + Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)
+              end
+            else
+              # remark is empty, fallback to resource_docs_cache
+              resource_docs_dir = resource_docs_cache[description['linkend']]
+              if resource_docs_dir
+                resource_docs_file = File.join(stepmod_dir, 'data/resource_docs', resource_docs_dir, 'resource.xml')
+                resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
+                schema = resource_docs.xpath("schema[@name='#{description['linkend']}']")
+
+                Dir.chdir(File.dirname(descriptions_file)) do
+                  wrapper = "<resource>#{schema.to_s}</resource>"
+                  output_express << "\n" + Stepmod::Utils::SmrlResourceConverter.convert(wrapper)
+                end
+              end
+            end
+          end
+        end
+
+        output_express
+      end
+    end
+  end
+end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.1.6"
+    VERSION = "0.2.4"
   end
 end

data/migrating_from_cvs.adoc

@@ -0,0 +1,190 @@
+= STEPmod CVS to Git import
+
+== Purpose
+
+The `cvs/` submodule is a CSV import of STEPmod files from the CVS server hosted at Boost Conseil.
+
+The goal is to import the CSV-managed files in a way useable via Git, with full history information.
+
+This guide is used as reference for the usage of `cvs-fast-import`:
+https://oitofelix.github.io/article-savannah-cvs-to-git-migration/
+
+
+== Strategy to import
+
+Cloning a remote CVS repository while importing is super slow,
+especially for a large repository like STEPmod.
+
+We have tested and settled on these steps:
+
+. Maintain a local `rsync` copy of the CVS repository.
+. Resolve all names from the CVS repository (CVS only stores UNIX usernames, in Git are names and emails) using `cvs-fast-import`.
+. Run `cvs-fast-import` to import the CVS repository into the `iso-10303-stepmod-cvs` Git repo.
+
+
+== Creating the `rsync` clone of the CVS repository
+
+[source,sh]
+----
+rsync -avrPz -e ssh ronald@cvs.boost-lab.net:/stepmod/ stepmod-rsync/
+----
+
+
+== Install cvs-fast-export
+
+`cvs-fast-export` only works on Linux.
+Run it on Ubuntu with the `rsync`'ed CVS directory.
+
+Install:
+[source,sh]
+----
+$ apt-get -y install cvs-fast-export
+----
+
+== Getting all users for email mapping
+
+Find all authors in the CVS repository using `cvs-fast-export -a`.
+It is much faster than using the equivalent CVS command to list all authors.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -a
+----
+
+
+== Create fast-import file for Git
+
+After all authors are mapped, run `cvs-fast-export` to create
+the import file.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -A author-map.txt > fast-import-file
+----
+
+== Perform the Git fast-import
+
+Once the fast-import file is created, we can perform the Git import.
+
+[source,sh]
+----
+$ cd iso-10303-stepmod-cvs
+$ git fast-import < ../fast-import-file
+----
+
+
+== Upload the new Git repo
+
+[source,sh]
+----
+git push --all && git push --tags
+----
+
+
+== DEPRECATED steps using `git cvsimport` (do not use, it won't work)
+
+=== General
+
+https://stackoverflow.com/questions/11362676/how-to-import-and-keep-updated-a-cvs-repository-in-git[This StackOverflow post]
+describes steps for using `git cvsimport`.
+
+Originally the `git cvsimport` tool was chosen since it is part of
+`git`. However, it utilizes a deprecated/unmaintained tool called
+`cvsps`. The latest `cvsps` is version 3, but only version 2 is
+compatible with `git cvsimport`.
+
+In addition, the `cvsps` tool is maintained by the maintainer
+of `cvs-fast-import`, and is no longer updated.
+Eventually `cvs-fast-import` is used instead.
+
+WARNING: This command completely fails
+on this repository because it is too large and complex.
+
+
+=== Setup
+
+On macOS, run the following commands to setup for running the import. The `git` executable must be installed.
+
+Install `cvsps` version 2.
+
+NOTE: The steps from the StackOverflow of installing `cvsps` no longer work.
+
+[source,sh]
+----
+$ brew tap Frizlab/Perso
+# ==> Tapping frizlab/perso
+# Cloning into '/usr/local/Homebrew/Library/Taps/frizlab/homebrew-perso'...
+# remote: Enumerating objects: 123, done.
+# remote: Total 123 (delta 0), reused 0 (delta 0), pack-reused 123
+# Receiving objects: 100% (123/123), 19.08 KiB | 91.00 KiB/s, done.
+# Resolving deltas: 100% (43/43), done.
+# Tapped 1 cask and 10 formulae (38 files, 60.5KB).
+
+$ brew install cvsps@2
+# ==> Installing cvsps@2 from frizlab/perso
+# Warning: A newer Command Line Tools release is available.
+# Update them from Software Update in System Preferences or
+# https://developer.apple.com/download/more/.
+# ==> Downloading https://deb.debian.org/debian/pool/main/c/cvsps/cvsps_2.1.orig.tar.gz
+######################################################################## 100.0%
+# ==> make all
+# ==> make install prefix=/usr/local/Cellar/cvsps@2/2.1
+# 🍺  /usr/local/Cellar/cvsps@2/2.1: 7 files, 124.6KB, built in 6 seconds
+----
+
+Verify it is installed:
+[source,sh]
+----
+$ cvsps -v
+# Can't open CVS/Root
+# cannot determine CVSROOT
+----
+
+////
+== Checkout the CVS repository
+
+First set the `CVSROOT` and `CVS_RSH` variables.
+
+[source,sh]
+----
+export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+export CVS_RSH=ssh
+cvs checkout stepmod
+# => stepmod/ is created in $PWD
+----
+////
+
+
+=== Run the import
+
+Run the import from CVS to Git.
+
+Go to the directory that will carry the target Git repository.
+
+Run `git cvsimport`.
+
+[source,sh]
+----
+# Importing from remote
+$ export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+$ export CVS_RSH=ssh
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+
+# Importing from local rsync'ed copy
+$ export CVSROOT=$(pwd)/stepmod-rsync
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+----
+
+WARNING: TLDR. Technically this should work, but I ran into a `cvsps cannot allocate memory` error with 64GB of RAM, and not even completing the clone after 24 hours. So I gave up and switched to local. And local still takes a long time with tons of error messages. `cvs-fast-import` only takes 5-10 minutes to import.
+
+
+=== Updating the CVS import
+
+Run `git cvsimport` to synchronize the Git repo using updated data from CVS.
+
+[source,sh]
+----
+$ git cvsimport
+----
+
+

data/stepmod-utils.gemspec

@@ -31,5 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "thor", ">= 0.20.3"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.2.9"
+  spec.add_runtime_dependency "ptools", '~> 1.3'
+  spec.add_runtime_dependency "concurrent-ruby"
   spec.add_development_dependency "byebug", "~> 11.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-06 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.9
+- !ruby/object:Gem::Dependency
+  name: ptools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -54,44 +82,92 @@ dependencies:
         version: '11.1'
 description: 'Stepmod-utils is a toolkit that works on STEPmod data.
 
-  '
+'
 email:
 - open.source@ribose.com
 executables:
+- stepmod-annotate
+- stepmod-annotate-all
+- stepmod-build-resource-docs-cache
+- stepmod-convert-express-description
+- stepmod-convert-express-resource
 - stepmod-extract-terms
+- stepmod-find-express-files
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/macos.yml"
+- ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
-- ".github/workflows/ubuntu.yml"
-- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".rspec"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Makefile
 - README.adoc
 - Rakefile
 - bin/console
 - bin/setup
+- exe/stepmod-annotate
+- exe/stepmod-annotate-all
+- exe/stepmod-build-resource-docs-cache
+- exe/stepmod-convert-express-description
+- exe/stepmod-convert-express-resource
 - exe/stepmod-extract-terms
+- exe/stepmod-find-express-files
 - lib/stepmod/utils.rb
 - lib/stepmod/utils/bibdata.rb
+- lib/stepmod/utils/cleaner.rb
 - lib/stepmod/utils/concept.rb
+- lib/stepmod/utils/converters/a.rb
+- lib/stepmod/utils/converters/blockquote.rb
+- lib/stepmod/utils/converters/br.rb
+- lib/stepmod/utils/converters/bypass.rb
 - lib/stepmod/utils/converters/clause_ref.rb
+- lib/stepmod/utils/converters/code.rb
+- lib/stepmod/utils/converters/comment.rb
+- lib/stepmod/utils/converters/dd.rb
 - lib/stepmod/utils/converters/def.rb
 - lib/stepmod/utils/converters/definition.rb
+- lib/stepmod/utils/converters/dl.rb
+- lib/stepmod/utils/converters/drop.rb
+- lib/stepmod/utils/converters/dt.rb
 - lib/stepmod/utils/converters/em.rb
+- lib/stepmod/utils/converters/em_express_description.rb
+- lib/stepmod/utils/converters/eqn.rb
 - lib/stepmod/utils/converters/example.rb
+- lib/stepmod/utils/converters/express_g.rb
 - lib/stepmod/utils/converters/express_ref.rb
+- lib/stepmod/utils/converters/express_ref_express_description.rb
+- lib/stepmod/utils/converters/ext_description.rb
+- lib/stepmod/utils/converters/ext_descriptions.rb
+- lib/stepmod/utils/converters/fund_cons.rb
+- lib/stepmod/utils/converters/head.rb
+- lib/stepmod/utils/converters/hr.rb
+- lib/stepmod/utils/converters/ignore.rb
+- lib/stepmod/utils/converters/introduction.rb
 - lib/stepmod/utils/converters/module_ref.rb
 - lib/stepmod/utils/converters/note.rb
 - lib/stepmod/utils/converters/ol.rb
+- lib/stepmod/utils/converters/p.rb
+- lib/stepmod/utils/converters/pass_through.rb
+- lib/stepmod/utils/converters/q.rb
+- lib/stepmod/utils/converters/resource.rb
+- lib/stepmod/utils/converters/schema.rb
+- lib/stepmod/utils/converters/schema_diag.rb
 - lib/stepmod/utils/converters/stem.rb
+- lib/stepmod/utils/converters/strong.rb
+- lib/stepmod/utils/converters/sub.rb
+- lib/stepmod/utils/converters/sup.rb
 - lib/stepmod/utils/converters/synonym.rb
 - lib/stepmod/utils/converters/term.rb
+- lib/stepmod/utils/converters/text.rb
+- lib/stepmod/utils/html_to_asciimath.rb
+- lib/stepmod/utils/smrl_description_converter.rb
+- lib/stepmod/utils/smrl_resource_converter.rb
 - lib/stepmod/utils/stepmod_definition_converter.rb
+- lib/stepmod/utils/stepmod_file_annotator.rb
 - lib/stepmod/utils/version.rb
+- migrating_from_cvs.adoc
 - resource_example.xml
 - stepmod-utils.gemspec
 homepage: https://github.com/metanorma/stepmod-utils