stepmod-utils 0.1.8 → 0.2.6

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.1.8"
+    VERSION = "0.2.6"
   end
 end

data/migrating_from_cvs.adoc

@@ -0,0 +1,190 @@
+= STEPmod CVS to Git import
+
+== Purpose
+
+The `cvs/` submodule is a CSV import of STEPmod files from the CVS server hosted at Boost Conseil.
+
+The goal is to import the CSV-managed files in a way useable via Git, with full history information.
+
+This guide is used as reference for the usage of `cvs-fast-import`:
+https://oitofelix.github.io/article-savannah-cvs-to-git-migration/
+
+
+== Strategy to import
+
+Cloning a remote CVS repository while importing is super slow,
+especially for a large repository like STEPmod.
+
+We have tested and settled on these steps:
+
+. Maintain a local `rsync` copy of the CVS repository.
+. Resolve all names from the CVS repository (CVS only stores UNIX usernames, in Git are names and emails) using `cvs-fast-import`.
+. Run `cvs-fast-import` to import the CVS repository into the `iso-10303-stepmod-cvs` Git repo.
+
+
+== Creating the `rsync` clone of the CVS repository
+
+[source,sh]
+----
+rsync -avrPz -e ssh ronald@cvs.boost-lab.net:/stepmod/ stepmod-rsync/
+----
+
+
+== Install cvs-fast-export
+
+`cvs-fast-export` only works on Linux.
+Run it on Ubuntu with the `rsync`'ed CVS directory.
+
+Install:
+[source,sh]
+----
+$ apt-get -y install cvs-fast-export
+----
+
+== Getting all users for email mapping
+
+Find all authors in the CVS repository using `cvs-fast-export -a`.
+It is much faster than using the equivalent CVS command to list all authors.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -a
+----
+
+
+== Create fast-import file for Git
+
+After all authors are mapped, run `cvs-fast-export` to create
+the import file.
+
+[source,sh]
+----
+$ find stepmod-rsync -type f | cvs-fast-export -A author-map.txt > fast-import-file
+----
+
+== Perform the Git fast-import
+
+Once the fast-import file is created, we can perform the Git import.
+
+[source,sh]
+----
+$ cd iso-10303-stepmod-cvs
+$ git fast-import < ../fast-import-file
+----
+
+
+== Upload the new Git repo
+
+[source,sh]
+----
+git push --all && git push --tags
+----
+
+
+== DEPRECATED steps using `git cvsimport` (do not use, it won't work)
+
+=== General
+
+https://stackoverflow.com/questions/11362676/how-to-import-and-keep-updated-a-cvs-repository-in-git[This StackOverflow post]
+describes steps for using `git cvsimport`.
+
+Originally the `git cvsimport` tool was chosen since it is part of
+`git`. However, it utilizes a deprecated/unmaintained tool called
+`cvsps`. The latest `cvsps` is version 3, but only version 2 is
+compatible with `git cvsimport`.
+
+In addition, the `cvsps` tool is maintained by the maintainer
+of `cvs-fast-import`, and is no longer updated.
+Eventually `cvs-fast-import` is used instead.
+
+WARNING: This command completely fails
+on this repository because it is too large and complex.
+
+
+=== Setup
+
+On macOS, run the following commands to setup for running the import. The `git` executable must be installed.
+
+Install `cvsps` version 2.
+
+NOTE: The steps from the StackOverflow of installing `cvsps` no longer work.
+
+[source,sh]
+----
+$ brew tap Frizlab/Perso
+# ==> Tapping frizlab/perso
+# Cloning into '/usr/local/Homebrew/Library/Taps/frizlab/homebrew-perso'...
+# remote: Enumerating objects: 123, done.
+# remote: Total 123 (delta 0), reused 0 (delta 0), pack-reused 123
+# Receiving objects: 100% (123/123), 19.08 KiB | 91.00 KiB/s, done.
+# Resolving deltas: 100% (43/43), done.
+# Tapped 1 cask and 10 formulae (38 files, 60.5KB).
+
+$ brew install cvsps@2
+# ==> Installing cvsps@2 from frizlab/perso
+# Warning: A newer Command Line Tools release is available.
+# Update them from Software Update in System Preferences or
+# https://developer.apple.com/download/more/.
+# ==> Downloading https://deb.debian.org/debian/pool/main/c/cvsps/cvsps_2.1.orig.tar.gz
+######################################################################## 100.0%
+# ==> make all
+# ==> make install prefix=/usr/local/Cellar/cvsps@2/2.1
+# 🍺  /usr/local/Cellar/cvsps@2/2.1: 7 files, 124.6KB, built in 6 seconds
+----
+
+Verify it is installed:
+[source,sh]
+----
+$ cvsps -v
+# Can't open CVS/Root
+# cannot determine CVSROOT
+----
+
+////
+== Checkout the CVS repository
+
+First set the `CVSROOT` and `CVS_RSH` variables.
+
+[source,sh]
+----
+export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+export CVS_RSH=ssh
+cvs checkout stepmod
+# => stepmod/ is created in $PWD
+----
+////
+
+
+=== Run the import
+
+Run the import from CVS to Git.
+
+Go to the directory that will carry the target Git repository.
+
+Run `git cvsimport`.
+
+[source,sh]
+----
+# Importing from remote
+$ export CVSROOT=:ext:ronald@cvs.boost-lab.net:/stepmod
+$ export CVS_RSH=ssh
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+
+# Importing from local rsync'ed copy
+$ export CVSROOT=$(pwd)/stepmod-rsync
+$ git cvsimport -C iso-10303-stepmod-cvs -r cvs -k -v -d $CVSROOT stepmod
+----
+
+WARNING: TLDR. Technically this should work, but I ran into a `cvsps cannot allocate memory` error with 64GB of RAM, and not even completing the clone after 24 hours. So I gave up and switched to local. And local still takes a long time with tons of error messages. `cvs-fast-import` only takes 5-10 minutes to import.
+
+
+=== Updating the CVS import
+
+Run `git cvsimport` to synchronize the Git repo using updated data from CVS.
+
+[source,sh]
+----
+$ git cvsimport
+----
+
+

data/stepmod-utils.gemspec

@@ -31,5 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "thor", ">= 0.20.3"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.2.9"
+  spec.add_runtime_dependency "ptools", '~> 1.3'
+  spec.add_runtime_dependency "concurrent-ruby"
   spec.add_development_dependency "byebug", "~> 11.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.6
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-07 00:00:00.000000000 Z
+date: 2021-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.9
+- !ruby/object:Gem::Dependency
+  name: ptools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -58,47 +86,94 @@ description: 'Stepmod-utils is a toolkit that works on STEPmod data.
 email:
 - open.source@ribose.com
 executables:
+- stepmod-annotate-all
+- stepmod-build-resource-docs-cache
+- stepmod-convert-express-description
+- stepmod-convert-express-resource
 - stepmod-extract-terms
+- stepmod-find-express-files
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/macos.yml"
+- ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
-- ".github/workflows/ubuntu.yml"
-- ".github/workflows/windows.yml"
 - ".gitignore"
 - ".rspec"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Makefile
 - README.adoc
 - Rakefile
 - bin/console
 - bin/setup
+- exe/stepmod-annotate-all
+- exe/stepmod-build-resource-docs-cache
+- exe/stepmod-convert-express-description
+- exe/stepmod-convert-express-resource
 - exe/stepmod-extract-terms
+- exe/stepmod-find-express-files
 - lib/stepmod/utils.rb
 - lib/stepmod/utils/bibdata.rb
+- lib/stepmod/utils/cleaner.rb
 - lib/stepmod/utils/concept.rb
+- lib/stepmod/utils/converters/a.rb
+- lib/stepmod/utils/converters/blockquote.rb
+- lib/stepmod/utils/converters/br.rb
+- lib/stepmod/utils/converters/bypass.rb
 - lib/stepmod/utils/converters/clause_ref.rb
+- lib/stepmod/utils/converters/code.rb
+- lib/stepmod/utils/converters/comment.rb
+- lib/stepmod/utils/converters/dd.rb
 - lib/stepmod/utils/converters/def.rb
 - lib/stepmod/utils/converters/definition.rb
+- lib/stepmod/utils/converters/dl.rb
+- lib/stepmod/utils/converters/drop.rb
+- lib/stepmod/utils/converters/dt.rb
 - lib/stepmod/utils/converters/em.rb
+- lib/stepmod/utils/converters/em_express_description.rb
+- lib/stepmod/utils/converters/eqn.rb
 - lib/stepmod/utils/converters/example.rb
+- lib/stepmod/utils/converters/express_g.rb
 - lib/stepmod/utils/converters/express_ref.rb
+- lib/stepmod/utils/converters/express_ref_express_description.rb
+- lib/stepmod/utils/converters/ext_description.rb
+- lib/stepmod/utils/converters/ext_descriptions.rb
+- lib/stepmod/utils/converters/fund_cons.rb
+- lib/stepmod/utils/converters/head.rb
+- lib/stepmod/utils/converters/hr.rb
+- lib/stepmod/utils/converters/ignore.rb
+- lib/stepmod/utils/converters/introduction.rb
 - lib/stepmod/utils/converters/module_ref.rb
 - lib/stepmod/utils/converters/note.rb
 - lib/stepmod/utils/converters/ol.rb
+- lib/stepmod/utils/converters/p.rb
+- lib/stepmod/utils/converters/pass_through.rb
+- lib/stepmod/utils/converters/q.rb
+- lib/stepmod/utils/converters/resource.rb
+- lib/stepmod/utils/converters/schema.rb
+- lib/stepmod/utils/converters/schema_diag.rb
 - lib/stepmod/utils/converters/stem.rb
+- lib/stepmod/utils/converters/strong.rb
+- lib/stepmod/utils/converters/sub.rb
+- lib/stepmod/utils/converters/sup.rb
 - lib/stepmod/utils/converters/synonym.rb
+- lib/stepmod/utils/converters/table.rb
 - lib/stepmod/utils/converters/term.rb
+- lib/stepmod/utils/converters/text.rb
+- lib/stepmod/utils/html_to_asciimath.rb
+- lib/stepmod/utils/smrl_description_converter.rb
+- lib/stepmod/utils/smrl_resource_converter.rb
 - lib/stepmod/utils/stepmod_definition_converter.rb
+- lib/stepmod/utils/stepmod_file_annotator.rb
 - lib/stepmod/utils/version.rb
+- migrating_from_cvs.adoc
 - resource_example.xml
 - stepmod-utils.gemspec
 homepage: https://github.com/metanorma/stepmod-utils
 licenses:
 - BSD-2-Clause
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -113,8 +188,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.
 test_files: []

data/.github/workflows/macos.yml

@@ -1,39 +0,0 @@
-# Auto-generated by Cimas: Do not edit it manually!
-# See https://github.com/metanorma/cimas
-name: macos
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    paths-ignore:
-      - .github/workflows/ubuntu.yml
-      - .github/workflows/windows.yml
-
-jobs:
-  test-macos:
-    name: Test on Ruby ${{ matrix.ruby }} macOS
-    runs-on: macos-latest
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '2.6', '2.5', '2.4' ]
-        experimental: [false]
-        include:
-          - ruby: '2.7'
-            experimental: true
-    steps:
-      - uses: actions/checkout@master
-      - name: Use Ruby
-        uses: actions/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
-      - name: Update gems
-        run: |
-          sudo gem install bundler --force
-          bundle install --jobs 4 --retry 3
-      - name: Run specs
-        run: |
-          bundle exec rake

data/.github/workflows/ubuntu.yml

@@ -1,53 +0,0 @@
-# Auto-generated by Cimas: Do not edit it manually!
-# See https://github.com/metanorma/cimas
-name: ubuntu
-
-on:
-  push:
-    branches: [ master ]
-    tags:
-      - '*'
-  pull_request:
-    paths-ignore:
-      - .github/workflows/macos.yml
-      - .github/workflows/windows.yml
-
-jobs:
-  test-linux:
-    name: Test on Ruby ${{ matrix.ruby }} Ubuntu
-    runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '2.6', '2.5', '2.4' ]
-        experimental: [false]
-        include:
-          - ruby: '2.7'
-            experimental: true
-    steps:
-      - uses: actions/checkout@master
-      - name: Use Ruby
-        uses: actions/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
-      - name: Update gems
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3
-      - name: Run specs
-        run: |
-          bundle exec rake
-      - name: Trigger dependent repositories
-        if: github.ref == 'refs/heads/master' && matrix.ruby == '2.6'
-        env:
-          GH_USERNAME: ${{ secrets.PAT_USERNAME }}
-          GH_ACCESS_TOKEN: ${{ secrets.PAT_TOKEN }}
-        run: |
-          curl -LO --retry 3 https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/trigger-gh-actions.sh
-          [[ -f ".github/workflows/dependent_repos.env" ]] && source .github/workflows/dependent_repos.env
-          for repo in $DEPENDENT_REPOS
-          do
-            sh trigger-gh-actions.sh $ORGANISATION $repo $GH_USERNAME $GH_ACCESS_TOKEN $GITHUB_REPOSITORY "{ \"ref\": \"${GITHUB_REF}\" }"
-          done