stepmod-utils 0.3.10 → 0.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c31735350e54a2aa7f6de9b0de749126887d48a8e1ce23fd56378c7a5319641
-  data.tar.gz: aa494232f4f3858ea26f3379f602888066a7d1b98abd6d3c5cccc415d24fc58b
+  metadata.gz: 2dbfad6cebbd4552c2b91cfd5aac3dc5a032bde8a44269b83303d8472bf86310
+  data.tar.gz: 185a292f76f2e5b792383b8f2ca17438cb939a557f3df763ed5c9321c28b7438
 SHA512:
-  metadata.gz: cb202b0ed3ca8ccc4a21c36e68a4135b06d2b136227caf9b5d6627dc706d85ab7b0c1396582f5d71f967e3a35c11884a6dc30256e91fedae0a0e2d9ca52c5db9
-  data.tar.gz: 8a221bd8da7e75cf5653dee02fe484fbf1ab648ac8a0929941497d170310a5585b506e769c2f7ee695d3f694848aab1c874e83a668cdf397db6751d6ee3ca0b9
+  metadata.gz: a6f855167c151253c8b2278dfafe460198fe2c7789ce3cb2eba4dca20dc216546d88b68f199f48a6508574b1c28eaf476604799c99b3762662269eec743e0d5e
+  data.tar.gz: a85e8f5a2d4607f5c001c5889c4549d2698279bb5163b3e87c79e5e28c3f0bbea7a38dd56d3440af3b43467ac2d93d7b16407d63f27c7abc61ca3fe1f5aa6551

data/README.adoc

@@ -10,7 +10,8 @@ image:https://img.shields.io/github/commits-since/metanorma/stepmod-utils/latest
 
 == Purpose
 
-The `stepmod-utils` Ruby gem provides a number of tools to work with the STEPmod repository.
+The `stepmod-utils` Ruby gem provides a number of tools to work with the STEPmod
+repository.
 
 
 == CVS to Git migration procedures
@@ -31,11 +32,8 @@ Or include it in your gemspec.
 
 [source,sh]
 ----
-# Extracts from current directory
-$ stepmod-extract-terms
-
-# Extracts from specified stepmod/ or stepmod/data/ directory
-$ stepmod-extract-terms {stepmod-data-directory}
+# Extracts from specified stepmod/data/ directory
+$ stepmod-extract-terms -p {stepmod-data-directory} -i {path-to-repository_index.xml}
 ----
 
 Then these files will be created:

data/exe/stepmod-extract-terms

@@ -21,14 +21,48 @@ end
 require "bundler/setup"
 require "stepmod/utils/terms_extractor"
 
-stepmod_dir = ARGV.first || Dir.pwd
+require 'optparse'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+
+  opts.on("-p", "--path STEPMOD_DATA_PATH", String, "Path to STEPmod CVS data directory") do |path|
+    options[:stepmod_dir] = path
+  end
+
+  opts.on("-i", "--index INDEX_PATH", String, "Path to repository_index.xml") do |path|
+    unless path.nil?
+      options[:index_path] = Pathname.new(path).to_s
+    end
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+stepmod_dir = options[:stepmod_dir]
+if stepmod_dir.nil?
+  raise StandardError.new("STEPmod data path not set, set with the `-p` option.")
+else
+  log "STEPmod data path: `#{stepmod_dir}`"
+end
+
+index_path = options[:index_path] || File.join(stepmod_dir, "repository_index.xml")
+unless File.exists?(index_path)
+  raise StandardError.new("Index file not present at #{index_path}, set with the `-i` option.")
+else
+  log "Repository index path: `#{index_path}`"
+end
 
 general_concepts,
 resource_concepts,
 parsed_bibliography,
 part_concepts,
 part_resources,
-part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir)
+part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir, index_path)
 
 def part_to_title(bibdata)
   case bibdata.part.to_i
@@ -87,7 +121,7 @@ end
 
 File.open("03x-stepmod.adoc", "w") do |file|
   part_concepts.sort_by do |(bibdata, current_part_concepts)|
-    bibdata.part
+    bibdata.part.to_i
   end.each do |(bibdata, current_part_concepts)|
     fn = "03x-stepmod-#{bibdata.part}.adoc"
     file.puts("\ninclude::#{fn}[]\n")
@@ -102,7 +136,7 @@ part_resources.each do |(bibdata, current_part_resources)|
   end
   fn = "04x-stepmod-entities-resources-#{bibdata.part}.adoc"
   File.open(fn, "w") do |file|
-    file.puts("== #{part_to_title(bibdata)}\n\n")
+    # file.puts("== #{part_to_title(bibdata)}\n\n")
     file.puts(replace_images(current_part_resources.map(&:to_mn_adoc).join("\n")))
   end
   log "INFO: written to: #{fn}"
@@ -110,7 +144,7 @@ end
 
 File.open("04x-stepmod-entities-resources.adoc", "w") do |file|
   part_resources.sort_by do |(bibdata, current_part_resources)|
-    bibdata.part
+    bibdata.part.to_i
   end.each do |(bibdata, current_part_resources)|
     fn = "04x-stepmod-entities-resources-#{bibdata.part}.adoc"
     file.puts("\ninclude::#{fn}[]\n")
@@ -152,7 +186,7 @@ end
 
 File.open("05x-stepmod-entities-modules.adoc", "w") do |file|
   part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
-    bibdata.part
+    bibdata.part.to_i
   end.each do |(bibdata, part_modules_arm, part_modules_mim)|
     fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
     file.puts("\ninclude::#{fn}[]\n")

data/lib/stepmod/utils/cleaner.rb

@@ -4,7 +4,9 @@ module Stepmod
   module Utils
     class Cleaner < ReverseAdoc::Cleaner
       def tidy(string)
-        super.gsub(/^ +/, "")
+        super
+          .gsub(/^ +/, "")
+          .gsub(/\*\s([,.])/, '*\1') # remove space between * and comma or dot.
       end
     end
   end

data/lib/stepmod/utils/concept.rb

@@ -124,7 +124,7 @@ module Stepmod
 
       def to_mn_adoc
         <<~TEXT
-          // STEPmod path:#{!file_path.empty? ? " #{file_path}" : ''}
+          // STEPmod path:#{file_path.empty? ? '' : " #{file_path}"}
           #{converted_definition}
 
           [.source]

data/lib/stepmod/utils/converters/ext_description.rb

@@ -3,7 +3,7 @@ module Stepmod
     module Converters
       class ExtDescription < ReverseAdoc::Converters::Base
         def convert(node, state = {})
-          state = state.merge(schema_name: node["linkend"])
+          state = state.merge(schema_name: node["linkend"], non_flanking_whitesapce: true)
           child_text = treat_children(node, state).strip
 
           <<~TEMPLATE

data/lib/stepmod/utils/converters/stepmod_ext_description.rb

@@ -29,6 +29,13 @@ module Stepmod
             child_text = first_child
           end
 
+          # TEMP: Remove any whitespace (" ", not newlines) after an immediate
+          # newline due to:
+          # https://github.com/metanorma/iso-10303-2/issues/71
+          if child_text =~ /\n\ +/
+            child_text = child_text.gsub(/\n\ +/, "\n")
+          end
+
           # # Only taking the first sentence
           # if child_text.contains?(".")
           #   child_text = child_text.split(".").first
@@ -36,10 +43,10 @@ module Stepmod
 
           domain =  case linkend.first
                     when /_mim$/, /_arm$/
-                      "STEP module"
+                      "ISO 10303 application module"
                     # when /_schema$/
                     else
-                      "STEP resource"
+                      "ISO 10303 resource"
                     end
 
           <<~TEMPLATE

data/lib/stepmod/utils/converters/strong.rb

@@ -8,12 +8,13 @@ module Stepmod
 
         def convert(node, state = {})
           content = treat_children(node, state.merge(already_strong: true))
+          strong_tag = state[:non_flanking_whitesapce] ? '**' : '*'
           if content.strip.empty? || state[:already_strong]
             content
           else
             handle_express_escape_seq(
               node,
-              "#{content[/^\s*/]}*#{content.strip}*#{content[/\s*$/]}"
+              "#{content[/^\s*/]}#{strong_tag}#{content.strip}#{strong_tag}#{content[/\s*$/]}"
             )
           end
         end

data/lib/stepmod/utils/converters/uof.rb

@@ -14,7 +14,7 @@ module Stepmod
           # <<~TEXT
           # === #{node['name'].strip}
 
-          # <STEP module> #{treat_children(node, state).strip}
+          # <ISO 10303 application module> #{treat_children(node, state).strip}
           # TEXT
         end
       end

data/lib/stepmod/utils/stepmod_definition_converter.rb

@@ -17,6 +17,7 @@ require "stepmod/utils/converters/term"
 require "stepmod/utils/converters/synonym"
 require "stepmod/utils/converters/uof"
 require "stepmod/utils/converters/figure"
+require "stepmod/utils/cleaner"
 
 require "reverse_adoc/converters/a"
 require "reverse_adoc/converters/blockquote"
@@ -56,7 +57,7 @@ module Stepmod
                                                                      options)
           return "" unless result
 
-          ReverseAdoc.cleaner.tidy(result.dup)
+          Stepmod::Utils::Cleaner.new.tidy(result.dup)
         end
       end
     end

data/lib/stepmod/utils/terms_extractor.rb

@@ -15,6 +15,7 @@ module Stepmod
 
       attr_reader :stepmod_path,
                   :stepmod_dir,
+                  :index_path,
                   :general_concepts,
                   :resource_concepts,
                   :parsed_bibliography,
@@ -25,14 +26,15 @@ module Stepmod
                   :part_modules,
                   :stdout
 
-      def self.call(stepmod_dir, stdout = $stdout)
-        new(stepmod_dir, stdout).call
+      def self.call(stepmod_dir, index_path, stdout = $stdout)
+        new(stepmod_dir, index_path, stdout).call
       end
 
-      def initialize(stepmod_dir, stdout)
+      def initialize(stepmod_dir, index_path, stdout)
         @stdout = stdout
         @stepmod_dir = stepmod_dir
         @stepmod_path = Pathname.new(stepmod_dir).realpath
+        @index_path = Pathname.new(index_path).to_s
         @general_concepts = Glossarist::Collection.new
         @resource_concepts = Glossarist::Collection.new
         @parsed_bibliography = []
@@ -79,7 +81,7 @@ module Stepmod
 
         log "INFO: Detecting paths..."
 
-        repo_index = Nokogiri::XML(File.read(stepmod_path.join("repository_index.xml"))).root
+        repo_index = Nokogiri::XML(File.read(@index_path)).root
 
         files = []
 

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.10".freeze
+    VERSION = "0.3.14".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.14
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2021-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -215,7 +215,6 @@ files:
 - lib/stepmod/utils/stepmod_file_annotator.rb
 - lib/stepmod/utils/terms_extractor.rb
 - lib/stepmod/utils/version.rb
-- migrating_from_cvs.adoc
 - resource_example.xml
 - stepmod-utils.gemspec
 homepage: https://github.com/metanorma/stepmod-utils

data/migrating_from_cvs.adoc

@@ -1,190 +0,0 @@
-= STEPmod CVS to Git import
-
-== Purpose
-
-The `cvs/` submodule is a CSV import of STEPmod files from the CVS server hosted at Boost Conseil.
-
-The goal is to import the CSV-managed files in a way useable via Git, with full history information.
-
-This guide is used as reference for the usage of `cvs-fast-import`:
-https://oitofelix.github.io/article-savannah-cvs-to-git-migration/
-
-
-== Strategy to import
-
-Cloning a remote CVS repository while importing is super slow,
-especially for a large repository like STEPmod.
-
-We have tested and settled on these steps:
-
-. Maintain a local `rsync` copy of the CVS repository.
-. Resolve all names from the CVS repository (CVS only stores UNIX usernames, in Git are names and emails) using `cvs-fast-import`.
-. Run `cvs-fast-import` to import the CVS repository into the `iso-10303-stepmod-cvs` Git repo.
-
-
-== Creating the `rsync` clone of the CVS repository
-
-[source,sh]
-----
-rsync -avrPz -e ssh ronald@cvs.boost-lab.net:/stepmod/ stepmod-rsync/
-----
-
-
-== Install cvs-fast-export
-
-`cvs-fast-export` only works on Linux.
-Run it on Ubuntu with the `rsync`'ed CVS directory.
-
-Install:
-[source,sh]
-----
-$ apt-get -y install cvs-fast-export
-----
-
-== Getting all users for email mapping
-
-Find all authors in the CVS repository using `cvs-fast-export -a`.
-It is much faster than using the equivalent CVS command to list all authors.
-
-[source,sh]
-----
-$ find stepmod-rsync -type f | cvs-fast-export -a
-----
-
-
-== Create fast-import file for Git
-
-After all authors are mapped, run `cvs-fast-export` to create
-the import file.
-
-[source,sh]
-----
-$ find stepmod-rsync -type f | cvs-fast-export -A author-map.txt > fast-import-file
-----
-
-== Perform the Git fast-import
-
-Once the fast-import file is created, we can perform the Git import.
-
-[source,sh]
-----
-$ cd iso-10303-stepmod-cvs
-$ git fast-import < ../fast-import-file
-----
-
-
-== Upload the new Git repo
-
-[source,sh]
-----
-git push --all && git push --tags
-----
-
-
-== DEPRECATED steps using `git cvsimport` (do not use, it won't work)
-
-=== General
-
-https://stackoverflow.com/questions/11362676/how-to-import-and-keep-updated-a-cvs-repository-in-git[This StackOverflow post]
-describes steps for using `git cvsimport`.
-
-Originally the `git cvsimport` tool was chosen since it is part of
-`git`. However, it utilizes a deprecated/unmaintained tool called
-`cvsps`. The latest `cvsps` is version 3, but only version 2 is
-compatible with `git cvsimport`.
-
-In addition, the `cvsps` tool is maintained by the maintainer
-of `cvs-fast-import`, and is no longer updated.
-Eventually `cvs-fast-import` is used instead.
-
-WARNING: This command completely fails
-on this repository because it is too large and complex.
-
-
-=== Setup
-
-On macOS, run the following commands to setup for running the import. The `git` executable must be installed.
-
-Install `cvsps` version 2.
-
-NOTE: The steps from the StackOverflow of installing `cvsps` no longer work.
-
-[source,sh]
-----
-$ brew tap Frizlab/Perso
-# ==> Tapping frizlab/perso
-# Cloning into '/usr/local/Homebrew/Library/Taps/frizlab/homebrew-perso'...
-# remote: Enumerating objects: 123, done.
-# remote: Total 123 (delta 0), reused 0 (delta 0), pack-reused 123
-# Receiving objects: 100% (123/123), 19.08 KiB | 91.00 KiB/s, done.
-# Resolving deltas: 100% (43/43), done.
-# Tapped 1 cask and 10 formulae (38 files, 60.5KB).
-
-$ brew install cvsps@2
-# ==> Installing cvsps@2 from frizlab/perso
-# Warning: A newer Command Line Tools release is available.
-# Update them from Software Update in System Preferences or
-# https://developer.apple.com/download/more/.
-# ==> Downloading https://deb.debian.org/debian/pool/main/c/cvsps/cvsps_2.1.orig.tar.gz
-######################################################################## 100.0%
-# ==> make all
-# ==> make install prefix=/usr/local/Cellar/cvsps@2/2.1
-# 🍺  /usr/local/Cellar/cvsps@2/2.1: 7 files, 124.6KB, built in 6 seconds
-----
-
-Verify it is installed:
-[source,sh]
-----
-$ cvsps -v
-# Can't open CVS/Root
-# cannot determine CVSROOT
-----
-
-////
-== Checkout the CVS repository
-
-First set the `CVSROOT` and `CVS_RSH` variables.
-
-[source,sh]
-----
-export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
-export CVS_RSH=ssh
-cvs checkout stepmod
-# => stepmod/ is created in $PWD
-----
-////
-
-
-=== Run the import
-
-Run the import from CVS to Git.
-
-Go to the directory that will carry the target Git repository.
-
-Run `git cvsimport`.
-
-[source,sh]
-----
-# Importing from remote
-$ export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
-$ export CVS_RSH=ssh
-$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
-
-# Importing from local rsync'ed copy
-$ export CVSROOT=$(pwd)/stepmod-rsync
-$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
-----
-
-WARNING: TLDR. Technically this should work, but I ran into a `cvsps cannot allocate memory` error with 64GB of RAM, and not even completing the clone after 24 hours. So I gave up and switched to local. And local still takes a long time with tons of error messages. `cvs-fast-import` only takes 5-10 minutes to import.
-
-
-=== Updating the CVS import
-
-Run `git cvsimport` to synchronize the Git repo using updated data from CVS.
-
-[source,sh]
-----
-$ git cvsimport
-----
-
-