stepmod-utils 0.2.0 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 83672fac3acf9ef0d9a2ab17fffab1204b326c7d41018675c2cc819976ac8b40
-  data.tar.gz: 2a59c155d059ca7b73e48ebf5ddedac314e998dbb92eb301a55c460d8ac44e79
+  metadata.gz: fe9caf0ee63e637eb1a73289d7394a495b5f8a313c6ee3d69aff2bdcc150e57f
+  data.tar.gz: 791c199203956d117839a900869fd61362ead66703f6723cd4b5c0c8d871ccc4
 SHA512:
-  metadata.gz: ca130bac46780aeef8434443d68e02a83cfc2d17a5dd1634c13c6448fd33c1861e85cc3173303db64449e7da3465b1953ed344690894e09cd5698304c4b129eb
-  data.tar.gz: f4d94ac262f4c98ca810a1002c0bdc22d67cc3bd771521f4700e512386f226156d442de3c3f826c0aa7757bbc7d41a78dd42c22b166eca6a032b9b1177511424
+  metadata.gz: 1910feb532333a698daa864f4387ccefdbcdeb6ccbcf3a962d3e7ec5dfa0a0c0f17643e8eba22ed625aa2a308595b0c30ea7d49ce5bed2de0a8c638cb1e2e00d
+  data.tar.gz: 19757106a503652a64052eb823750bb5a618750b68bf42abdc3833dde6be9d3345dbc80c5051d37cea2507fe07c87b0669cf3af60c66cf6579ace1cd02f9bf96

data/.github/workflows/rake.yml

@@ -0,0 +1,42 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: rake
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  rake:
+    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        experimental: [ false ]
+        include:
+          - ruby: '3.0'
+            os: 'ubuntu-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'windows-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'macos-latest'
+            experimental: true
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: bundle exec rake

data/.github/workflows/release.yml

@@ -1,41 +1,44 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: release
 
 on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: |
+          Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
+        required: true
+        default: 'patch'
   push:
-    tags:
-      - '*'
+    tags: [ v* ]
 
 jobs:
   release:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - name: Use Ruby
-        uses: actions/setup-ruby@v1
+      - uses: actions/checkout@v2
+
+      - uses: ruby/setup-ruby@v1
         with:
           ruby-version: '2.6'
-          architecture: 'x64'
-      - name: Update gems
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3
-      - name: Update stepmod-utils.jar
-        run: |
-          rm -f bin/stepmod-utils.jar
-          rake bin/stepmod-utils.jar
-      - name: Run specs
+
+      - run: gem install gem-release
+
+      - if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
-          bundle exec rake
-      - name: Publish to rubygems.org
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          gem bump --version ${{ github.event.inputs.next_version }} --tag --push
+
+      - name: publish to rubygems.org
         env:
           RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}
         run: |
           gem install gem-release
-          touch ~/.gem/credentials
           cat > ~/.gem/credentials << EOF
           ---
           :rubygems_api_key: ${RUBYGEMS_API_KEY}
           EOF
           chmod 0600 ~/.gem/credentials
-          git status
           gem release

data/Gemfile

@@ -7,4 +7,5 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
-gem "pry"
+gem "pry"
+gem "concurrent-ruby", "~> 1.1"

data/Makefile

@@ -0,0 +1,5 @@
+convert_stepmod_repo:
+	bundle exec ./exe/stepmod-annotate-all $(10303_stepmod_path)
+
+create_svgs:
+	java -jar $(stepmod2mn_jar_path) $(10303_stepmod_path) -svg

data/README.adoc

@@ -1,4 +1,4 @@
-= stepmod-utils Ruby gem
+= STEPmod utilities
 
 image:https://img.shields.io/gem/v/metanorma.svg["Gem Version", link="https://rubygems.org/gems/metanorma"]
 image:https://github.com/metanorma/stepmod-utils/workflows/ubuntu/badge.svg["Ubuntu Build Status", link="https://github.com/metanorma/stepmod-utils/actions?query=workflow%3Aubuntu"]
@@ -12,6 +12,12 @@ image:https://img.shields.io/github/commits-since/metanorma/stepmod-utils/latest
 
 The `stepmod-utils` Ruby gem provides a number of tools to work with the STEPmod repository.
 
+
+== CVS to Git migration procedures
+
+include::migrating_from_cvs.adoc[]
+
+
 == Installation
 
 [source,ruby]
@@ -37,6 +43,81 @@ Then these files will be created:
 * `031-generated-terms.adoc` all terms extracted
 * `991-generated-bibliography.adoc` all bibliographic sources where the terms come from
 
+== Conversion of STEPmod descriptions.xml into adoc
+
+[source,sh]
+----
+$ stepmod-convert-express-description /path/to/the/stepmod/descriptions.xml
+----
+
+Example output:
+[source,adoc]
+-----
+(*"application_context_schema"
+
+*)
+(*"application_context_schema.exchange_identification_context_select"
+ The *exchange\_identification\_context\_select* type allows for the designation of the data types express_ref:[management_resources_schema:ir_express:management_resources_schema.identification_assignment] and express_ref:[person_organization_schema:ir_express:person_organization_schema.organization] .
+*)
+(*"application_context_schema.application_context"
+ An *application\_context*, as defined in ISO 10303-1, is a context in which product data is defined and has meaning. An *application\_context* represents various types of information that relate to product data and may affect the meaning and usage of that data.
+*)
+(*"application_context_schema.application_context.application"
+
+the express_ref:[support_resource_schema:ir_express:support_resource_schema.label] by which the *application\_context* is known.
+
+[NOTE]
+--
+The meaning of this attribute can be defined in the annotated EXPRESS schemas that use or specialize this entity, or in an agreement of common understanding between the partners sharing this information.
+--
+
+*)
+-----
+
+
+== Conversion of STEPmod resource.xml into adoc
+
+Given resource.xml file contents:
+
+[source,xml]
+----
+<schema name="contract_schema" number="8369" version="3">
+  <introduction>
+    The subject of the <b>contract_schema</b> is the description of contract agreements.
+  </introduction>
+  <fund_cons>
+    Contract information may be attached to any aspect of a product data.
+  </fund_cons>
+  <express-g>
+    <imgfile file="contract_schemaexpg1.xml"/>
+    <imgfile file="contract_schemaexpg2.xml"/>
+  </express-g>
+</schema>
+----
+
+Command:
+
+[source,sh]
+----
+$ stepmod-convert-express-resource resource.xml
+----
+
+Will give output:
+
+[source,adoc]
+----
+== Introduction
+
+The subject of the *contract_schema* is the description of contract agreements.
+
+== Fundamental concerns
+
+Contract information may be attached to any aspect of a product data.
+
+expg_image:contract_schemaexpg1.xml[]
+expg_image:contract_schemaexpg2.xml[]
+----
+
 
 == CVS revision detection
 

data/exe/stepmod-annotate-all

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'concurrent'
+require 'stepmod/utils/stepmod_file_annotator'
+
+stepmod_dir = ARGV.first || Dir.pwd
+
+# build resource_docs cache
+resource_docs_cache_file = `mktemp`
+`"#{File.join(__dir__, 'stepmod-build-resource-docs-cache')}" "#{stepmod_dir}" > "#{resource_docs_cache_file}"`
+
+# annotate each file
+files = `"#{File.join(__dir__, 'stepmod-find-express-files')}" "#{stepmod_dir}"`.strip.split("\n")
+MAX_THREADS = [2, Concurrent.processor_count].max * 2
+MAX_QUEUE_SIZE = MAX_THREADS * 4
+# https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
+pool = Concurrent::ThreadPoolExecutor.new(
+  :min_threads => MAX_THREADS,
+  :max_threads => MAX_THREADS,
+  :max_queue   => MAX_QUEUE_SIZE,
+  :fallback_policy => :caller_runs
+)
+files.each_slice(MAX_QUEUE_SIZE) do |batch|
+  puts("Queueing next batch")
+  batch.each do |file|
+    pool.post do
+      puts("#{Thread.current.object_id}: Queued processing #{file}")
+      annotated = Stepmod::Utils::StepmodFileAnnotator.new(express_file: file, resource_docs_cache_file: resource_docs_cache_file, stepmod_dir: stepmod_dir).call
+      File.open(File.join(File.dirname(file), "#{File.basename(file, '.exp')}_annotated.exp"), 'w') do |file|
+        file.puts(annotated)
+      end
+      puts("#{Thread.current.object_id}: Done processing #{file}")
+    end
+  end
+  pool.shutdown
+end
+
+# cleanup
+`rm "#{resource_docs_cache_file}"`

data/exe/stepmod-build-resource-docs-cache

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+require 'json'
+
+stepmod_dir = ARGV.first || Dir.pwd
+
+schemas = {}
+Dir.glob(File.join(stepmod_dir, 'data/resource_docs/*/resource.xml')).each do |resource_docs_file|
+  match = resource_docs_file.match('data/resource_docs/([^/]+)/resource.xml')
+  resource_docs_dir = match.captures[0]
+
+  resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
+  resource_docs.xpath('schema').each do |schema|
+    schemas[schema['name']] = resource_docs_dir
+  end
+end
+
+puts JSON.pretty_generate(schemas)

data/exe/stepmod-convert-express-description

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+class Gem::Specification
+  def this; self; end
+end
+
+require 'bundler/setup'
+require 'stepmod/utils/smrl_description_converter'
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+input_content = ARGF.read
+
+result = Stepmod::Utils::SmrlDescriptionConverter.convert(input_content, {})
+result.split("\n").each do |line|
+  begin
+    $stdout.puts(line)
+    $stdout.flush
+  rescue Errno::EPIPE
+    exit(74)
+  end
+end

data/exe/stepmod-convert-express-resource

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+class Gem::Specification
+  def this; self; end
+end
+
+require 'bundler/setup'
+require 'stepmod/utils/smrl_resource_converter'
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+input_content = ARGF.read
+
+result = Stepmod::Utils::SmrlResourceConverter.convert(input_content, {})
+result.split("\n").each do |line|
+  begin
+    $stdout.puts(line)
+    $stdout.flush
+  rescue Errno::EPIPE
+    exit(74)
+  end
+end

data/exe/stepmod-extract-terms

@@ -76,7 +76,7 @@ files.each do |file_path|
   end
 
   # TODO: we may want a command line option to override this in the future
-  unless %w(IS DIS TS).include? bibdata.doctype
+  unless %w(IS DIS FDIS TS).include? bibdata.doctype
     log "INFO: skipped #{bibdata.docid} as it is not IS, DIS or TS"
     next
   end
@@ -135,7 +135,7 @@ files.each do |file_path|
 
 end
 
-parsed_bibliography.uniq!
+parsed_bibliography
 
 File.open('031-generated-terms.adoc', 'w') { |file|
   file.puts(parsed_terms.map(&:to_mn_adoc).join("\n"))
@@ -144,7 +144,7 @@ File.open('031-generated-terms.adoc', 'w') { |file|
 log "INFO: written to: 031-generated-terms.adoc"
 
 File.open('991-generated-bibliography.adoc', 'w') { |file|
-  file.puts(parsed_bibliography.map(&:to_mn_adoc).join("\n"))
+  file.puts(parsed_bibliography.map(&:to_mn_adoc).uniq.join("\n"))
 }
 
 log "INFO: written to: 991-generated-bibliography.adoc"

data/exe/stepmod-find-express-files

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+
+stepmod_dir = ARGV.first || Dir.pwd
+
+index = Nokogiri::XML(File.read(File.join(stepmod_dir, 'repository_index.xml'))).root
+
+files = []
+index.xpath('modules/module').each do |item|
+  files << "#{stepmod_dir}/data/modules/#{item['name']}/arm.exp"
+  files << "#{stepmod_dir}/data/modules/#{item['name']}/mim.exp"
+end
+index.xpath('resources/resource').each do |item|
+  files << "#{stepmod_dir}/data/resources/#{item['name']}/#{item['name']}.exp"
+end
+index.xpath('business_object_models/business_object_model').each do |item|
+  files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/bom.exp"
+  files << "#{stepmod_dir}/data/business_object_models/#{item['name']}/DomainModel.exp"
+end
+
+existing_files = files.filter{|file| File.exists?(file)}
+puts existing_files

data/lib/stepmod/utils/cleaner.rb

@@ -0,0 +1,11 @@
+require 'reverse_adoc/cleaner'
+
+module Stepmod
+  module Utils
+    class Cleaner < ReverseAdoc::Cleaner
+      def tidy(string)
+        super.gsub(/^ +/, '')
+      end
+    end
+  end
+end

data/lib/stepmod/utils/converters/a.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Stepmod
+  module Utils
+    module Converters
+      class A < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          name  = treat_children(node, state)
+          href  = node['href']
+          title = extract_title(node)
+          id = node['id'] || node['name']
+
+          id = id&.gsub(/\s/, "")&.gsub(/__+/, "_")
+
+          if /^_Toc\d+$|^_GoBack$/.match id
+            ""
+          elsif !id.nil? && !id.empty?
+            "[[#{id}]]"
+          elsif href.to_s.start_with?('#')
+            href = href.sub(/^#/, "").gsub(/\s/, "").gsub(/__+/, "_")
+            if name.empty?
+              "<<#{href}>>"
+            else
+              "<<#{href},#{name}>>"
+            end
+          elsif href.to_s.empty?
+            name
+          else
+            name = title if name.empty?
+            href = "link:#{href}" unless href.to_s =~ URI::DEFAULT_PARSER.make_regexp
+            link = "#{href}[#{name}]"
+            # link.prepend(' ')
+            link = " #{link}"
+            link
+          end
+        end
+
+        private
+
+      end
+
+      ReverseAdoc::Converters.register :a, A.new
+    end
+  end
+end