stepmod-utils 0.1.7 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0234a977f2b4fe70773fb53f5a213858c45b217a127ac34d6497adee7bbc1324
-  data.tar.gz: 8d9c942f9c36ddca312712fcf4eae5f25b9c3080685d8d559bc19ac059e7c7a9
+  metadata.gz: 7d3544c6e6b2eca6cab5ab08050b9985b80f62d597604d6961cbb388a712160a
+  data.tar.gz: 6afde6fbd52b996a37a6d8bc88607adf97b760c0835d7eae5cee849a8911d8cb
 SHA512:
-  metadata.gz: bacc43cbc1ae0631ddac9e06c08f9ad61e5447acbeb4d55ee46a91203a363073efc96bf207c3c64ea04b4f0108726ff064a8a1e093620c40cd4f5da85d255624
-  data.tar.gz: 6bc94140c41c77370ba94048539fe2eae93cfa760272a3b36af97c40cfac233ec32d55425102ea75a92693a8058e0224e9fdebbe4eee910b45d33e6cfc295860
+  metadata.gz: c5631d5e7587e4d26d62425f623433147ad372f4f53ebc5d9f2e0c1414d0b84599b6655d242028fbc4df872637b2db2a23df31b905ddd48136a9add7de4d0a9b
+  data.tar.gz: d48868104b0fd44e74ec8459f9d45e1ed51b835b749bc3986c185c376d798ecb28da0d2a1813ebeb9faacc9e1432a0117adaa5101ade901e965424a101903bcc

data/.github/workflows/rake.yml

@@ -0,0 +1,42 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+name: rake
+
+on:
+  push:
+    branches: [ master, main ]
+    tags: [ v* ]
+  pull_request:
+
+jobs:
+  rake:
+    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        experimental: [ false ]
+        include:
+          - ruby: '3.0'
+            os: 'ubuntu-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'windows-latest'
+            experimental: true
+          - ruby: '3.0'
+            os: 'macos-latest'
+            experimental: true
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - run: bundle exec rake

data/.github/workflows/release.yml

@@ -1,41 +1,44 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
 name: release
 
 on:
+  workflow_dispatch:
+    inputs:
+      next_version:
+        description: |
+          Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
+        required: true
+        default: 'patch'
   push:
-    tags:
-      - '*'
+    tags: [ v* ]
 
 jobs:
   release:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - name: Use Ruby
-        uses: actions/setup-ruby@v1
+      - uses: actions/checkout@v2
+
+      - uses: ruby/setup-ruby@v1
         with:
           ruby-version: '2.6'
-          architecture: 'x64'
-      - name: Update gems
-        run: |
-          gem install bundler
-          bundle install --jobs 4 --retry 3
-      - name: Update stepmod-utils.jar
-        run: |
-          rm -f bin/stepmod-utils.jar
-          rake bin/stepmod-utils.jar
-      - name: Run specs
+
+      - run: gem install gem-release
+
+      - if: ${{ github.event_name == 'workflow_dispatch' }}
         run: |
-          bundle exec rake
-      - name: Publish to rubygems.org
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          gem bump --version ${{ github.event.inputs.next_version }} --tag --push
+
+      - name: publish to rubygems.org
         env:
           RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}
         run: |
           gem install gem-release
-          touch ~/.gem/credentials
           cat > ~/.gem/credentials << EOF
           ---
           :rubygems_api_key: ${RUBYGEMS_API_KEY}
           EOF
           chmod 0600 ~/.gem/credentials
-          git status
           gem release

data/Gemfile

@@ -7,3 +7,5 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+gem "pry"
+gem "concurrent-ruby", "~> 1.1"

data/Makefile

@@ -0,0 +1,5 @@
+convert_stepmod_repo:
+	bundle exec ./exe/stepmod-annotate-all $(10303_stepmod_path)
+
+create_svgs:
+	java -jar $(stepmod2mn_jar_path) $(10303_stepmod_path) -svg

data/README.adoc

@@ -1,4 +1,4 @@
-= stepmod-utils Ruby gem
+= STEPmod utilities
 
 image:https://img.shields.io/gem/v/metanorma.svg["Gem Version", link="https://rubygems.org/gems/metanorma"]
 image:https://github.com/metanorma/stepmod-utils/workflows/ubuntu/badge.svg["Ubuntu Build Status", link="https://github.com/metanorma/stepmod-utils/actions?query=workflow%3Aubuntu"]
@@ -10,7 +10,13 @@ image:https://img.shields.io/github/commits-since/metanorma/stepmod-utils/latest
 
 == Purpose
 
-The stepmod-utils Ruby gem provides a number of tools to work with the STEPmod repository/
+The `stepmod-utils` Ruby gem provides a number of tools to work with the STEPmod repository.
+
+
+== CVS to Git migration procedures
+
+include::migrating_from_cvs.adoc[]
+
 
 == Installation
 
@@ -37,4 +43,132 @@ Then these files will be created:
 * `031-generated-terms.adoc` all terms extracted
 * `991-generated-bibliography.adoc` all bibliographic sources where the terms come from
 
+== Conversion of STEPmod descriptions.xml into adoc
+
+[source,sh]
+----
+$ stepmod-convert-express-description /path/to/the/stepmod/descriptions.xml
+----
+
+Example output:
+[source,adoc]
+-----
+(*"application_context_schema"
+
+*)
+(*"application_context_schema.exchange_identification_context_select"
+ The *exchange\_identification\_context\_select* type allows for the designation of the data types express_ref:[management_resources_schema:ir_express:management_resources_schema.identification_assignment] and express_ref:[person_organization_schema:ir_express:person_organization_schema.organization] .
+*)
+(*"application_context_schema.application_context"
+ An *application\_context*, as defined in ISO 10303-1, is a context in which product data is defined and has meaning. An *application\_context* represents various types of information that relate to product data and may affect the meaning and usage of that data.
+*)
+(*"application_context_schema.application_context.application"
+
+the express_ref:[support_resource_schema:ir_express:support_resource_schema.label] by which the *application\_context* is known.
+
+[NOTE]
+--
+The meaning of this attribute can be defined in the annotated EXPRESS schemas that use or specialize this entity, or in an agreement of common understanding between the partners sharing this information.
+--
+
+*)
+-----
+
+
+== Conversion of STEPmod resource.xml into adoc
+
+Given resource.xml file contents:
+
+[source,xml]
+----
+<schema name="contract_schema" number="8369" version="3">
+  <introduction>
+    The subject of the <b>contract_schema</b> is the description of contract agreements.
+  </introduction>
+  <fund_cons>
+    Contract information may be attached to any aspect of a product data.
+  </fund_cons>
+  <express-g>
+    <imgfile file="contract_schemaexpg1.xml"/>
+    <imgfile file="contract_schemaexpg2.xml"/>
+  </express-g>
+</schema>
+----
+
+Command:
+
+[source,sh]
+----
+$ stepmod-convert-express-resource resource.xml
+----
+
+Will give output:
+
+[source,adoc]
+----
+== Introduction
+
+The subject of the *contract_schema* is the description of contract agreements.
+
+== Fundamental concerns
+
+Contract information may be attached to any aspect of a product data.
+
+expg_image:contract_schemaexpg1.xml[]
+expg_image:contract_schemaexpg2.xml[]
+----
+
+
+== CVS revision detection
+
+If the STEPmod repository is a CVS repository, it will display the
+working and repository revisions as comments.
+
+WARNING: CVS revision detection can be *slow*! Be patient.
+
+Sample output:
+
+[source,sh]
+----
+[stepmod-utils] INFO: STEPmod directory set to ../x/stepmod/.
+[stepmod-utils] INFO: STEPmod directory is a CVS repository and will detect revisions.
+[stepmod-utils] INFO: [CVS] Detecting file revisions can be slow, please be patient!
+[stepmod-utils] INFO: Detecting paths...
+[stepmod-utils] INFO: Processing XML file data/business_object_models/managed_model_based_3d_engineering/business_object_model.xml
+[stepmod-utils] INFO: Detecting CVS revision...
+[stepmod-utils] INFO: CVS working rev (1.148), repo rev (1.148)
+[stepmod-utils] INFO: Completed processing XML file data/business_object_models/managed_model_based_3d_engineering/business_object_model.xml
+[stepmod-utils] INFO: Processing XML file data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
+[stepmod-utils] INFO: Detecting CVS revision...
+[stepmod-utils] INFO: CVS working rev (1.31), repo rev (1.31)
+[stepmod-utils] INFO: Completed processing XML file data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
+...
+ stepmod/data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
+[stepmod-utils] INFO: written to: 031-generated-terms.adoc
+[stepmod-utils] INFO: written to: 991-generated-bibliography.adoc
+----
+
+
+== Non-CVS repository
+
+If the STEPmod repository is not a CVS repository, or if the `cvs` executable
+cannot be found, term extract will still work.
+
+Sample output:
+
+[source,sh]
+----
+[stepmod-utils] INFO: STEPmod directory set to ../iso-10303-stepmod.
+[stepmod-utils] INFO: STEPmod directory is not a CVS repository, skipping revision detection.
+[stepmod-utils] INFO: Detecting paths...
+[stepmod-utils] INFO: Processing XML file data/application_protocols/boundary_representation_for_iso_14306_open_jt/application_protocol.xml
+[stepmod-utils] INFO: skipped ISO/CD 10303-243 as it is not IS, DIS or TS
+[stepmod-utils] INFO: Processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
+[stepmod-utils] INFO: Completed processing XML file data/application_protocols/configuration_control_3d_design_ed2/application_protocol.xml
+[stepmod-utils] INFO: Processing XML file data/application_protocols/electronic_assembly_interconnect_and_packaging_design/application_protocol.xml
+[stepmod-utils] INFO: skipped ISO/CD 10303-210:2019 as it is not IS, DIS or TS
+[stepmod-utils] INFO: Processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
+[stepmod-utils] INFO: Completed processing XML file data/application_protocols/functional_data_and_their_schematic_representation_for_process_plant/application_protocol.xml
+...
+----
 

data/exe/stepmod-annotate-all

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'concurrent'
+require 'stepmod/utils/stepmod_file_annotator'
+
+stepmod_dir = ARGV.first || Dir.pwd
+
+# build resource_docs cache
+resource_docs_cache_file = `mktemp`
+`"#{File.join(__dir__, 'stepmod-build-resource-docs-cache')}" "#{stepmod_dir}" > "#{resource_docs_cache_file}"`
+
+# annotate each file
+files = `"#{File.join(__dir__, 'stepmod-find-express-files')}" "#{stepmod_dir}"`.strip.split("\n")
+MAX_THREADS = [2, Concurrent.processor_count].max * 2
+MAX_QUEUE_SIZE = MAX_THREADS * 4
+# https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/thread_pools.md
+pool = Concurrent::ThreadPoolExecutor.new(
+  :min_threads => MAX_THREADS,
+  :max_threads => MAX_THREADS,
+  :max_queue   => MAX_QUEUE_SIZE,
+  :fallback_policy => :caller_runs
+)
+files.each_slice(MAX_QUEUE_SIZE) do |batch|
+  puts("Queueing next batch")
+  batch.each do |file|
+    pool.post do
+      puts("#{Thread.current.object_id}: Queued processing #{file}")
+      annotated = Stepmod::Utils::StepmodFileAnnotator.new(express_file: file, resource_docs_cache_file: resource_docs_cache_file, stepmod_dir: stepmod_dir).call
+      File.open(File.join(File.dirname(file), "#{File.basename(file, '.exp')}_annotated.exp"), 'w') do |file|
+        file.puts(annotated)
+      end
+      puts("#{Thread.current.object_id}: Done processing #{file}")
+    end
+  end
+  pool.shutdown
+end
+
+# cleanup
+`rm "#{resource_docs_cache_file}"`

data/exe/stepmod-build-resource-docs-cache

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'nokogiri'
+require 'json'
+
+stepmod_dir = ARGV.first || Dir.pwd
+
+schemas = {}
+Dir.glob(File.join(stepmod_dir, 'data/resource_docs/*/resource.xml')).each do |resource_docs_file|
+  match = resource_docs_file.match('data/resource_docs/([^/]+)/resource.xml')
+  resource_docs_dir = match.captures[0]
+
+  resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
+  resource_docs.xpath('schema').each do |schema|
+    schemas[schema['name']] = resource_docs_dir
+  end
+end
+
+puts JSON.pretty_generate(schemas)

data/exe/stepmod-convert-express-description

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+class Gem::Specification
+  def this; self; end
+end
+
+require 'bundler/setup'
+require 'stepmod/utils/smrl_description_converter'
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+input_content = ARGF.read
+
+result = Stepmod::Utils::SmrlDescriptionConverter.convert(input_content, {})
+result.split("\n").each do |line|
+  begin
+    $stdout.puts(line)
+    $stdout.flush
+  rescue Errno::EPIPE
+    exit(74)
+  end
+end

data/exe/stepmod-convert-express-resource

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+# resolve bin path, ignoring symlinks
+require "pathname"
+bin_file = Pathname.new(__FILE__).realpath
+
+# add self to libpath
+$:.unshift File.expand_path("../../lib", bin_file)
+
+# Fixes https://github.com/rubygems/rubygems/issues/1420
+require "rubygems/specification"
+
+class Gem::Specification
+  def this; self; end
+end
+
+require 'bundler/setup'
+require 'stepmod/utils/smrl_resource_converter'
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+input_content = ARGF.read
+
+result = Stepmod::Utils::SmrlResourceConverter.convert(input_content, {})
+result.split("\n").each do |line|
+  begin
+    $stdout.puts(line)
+    $stdout.flush
+  rescue Errno::EPIPE
+    exit(74)
+  end
+end

data/exe/stepmod-extract-terms

@@ -1,8 +1,7 @@
-#!/usr/bin/env ruby
-
 #!/usr/bin/env ruby
 # encoding: UTF-8
 
+require 'pry'
 # resolve bin path, ignoring symlinks
 require "pathname"
 bin_file = Pathname.new(__FILE__).realpath
@@ -21,6 +20,7 @@ require 'bundler/setup'
 require 'stepmod/utils/stepmod_definition_converter'
 require 'stepmod/utils/bibdata'
 require 'stepmod/utils/concept'
+require 'ptools'
 
 ReverseAdoc.config.unknown_tags = :bypass
 
@@ -34,7 +34,22 @@ def log message
   puts "[stepmod-utils] #{message}"
 end
 
-log "STEPmod directory set to #{stepmod_dir}. Detecting paths..."
+stepmod_path = Pathname.new(stepmod_dir).realpath
+
+# If we are using the stepmod CVS repository, provide the revision number per file
+has_cvs = File.which("cvs")
+cvs_mode = has_cvs && Dir.exists?(stepmod_path.join('CVS'))
+
+log "INFO: STEPmod directory set to #{stepmod_dir}."
+
+if cvs_mode
+  log "INFO: STEPmod directory is a CVS repository and will detect revisions."
+  log "INFO: [CVS] Detecting file revisions can be slow, please be patient!"
+else
+  log "INFO: STEPmod directory is not a CVS repository, skipping revision detection."
+end
+
+log "INFO: Detecting paths..."
 
 files = %w(
   resource.xml
@@ -46,25 +61,48 @@ files = %w(
 end.flatten.sort.uniq
 
 files.each do |file_path|
-  log "Processing XML file #{file_path}"
+  file_path = Pathname.new(file_path).realpath
+  fpath = file_path.relative_path_from(stepmod_path)
+
+  log "INFO: Processing XML file #{fpath}"
   current_document = Nokogiri::XML(File.read(file_path)).root
 
   bibdata = nil
   begin
     bibdata = Stepmod::Utils::Bibdata.new(document: current_document)
   rescue
-    log "WARNING: Unknown file #{file_path}, skipped"
+    log "WARNING: Unknown file #{fpath}, skipped"
     next
   end
 
   # TODO: we may want a command line option to override this in the future
   unless %w(IS DIS TS).include? bibdata.doctype
-    log "WARNING: skipping #{bibdata.docid} as it is not IS, DIS or TS"
+    log "INFO: skipped #{bibdata.docid} as it is not IS, DIS or TS"
     next
   end
 
   if bibdata.part.to_s.empty?
-    log "FATAL: missing `part` attribute: #{file_path}"
+    log "FATAL: missing `part` attribute: #{fpath}"
+    log "INFO: skipped #{bibdata.docid} as it is missing `part` attribute."
+    next
+  end
+
+  revision_string = "\n// CVS: revision not detected"
+  if cvs_mode
+    # Run `cvs status` to find out version
+
+    log "INFO: Detecting CVS revision..."
+    Dir.chdir(stepmod_path) do
+      status = `cvs status #{fpath}`
+
+      unless status.empty?
+        working_rev = status.split(/\n/).grep(/Working revision:/).first.match(/revision:\s+(.+)$/)[1]
+        repo_rev = status.split(/\n/).grep(/Repository revision:/).first.match(/revision:\t(.+)\t/)[1]
+        log "INFO: CVS working rev (#{working_rev}), repo rev (#{repo_rev})"
+        revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
+          "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
+      end
+    end
   end
 
   # read definitions
@@ -73,23 +111,28 @@ files.each do |file_path|
     term_id = definition['id']
     unless term_id.nil?
       if encountered_terms[term_id]
-        log "FATAL: Duplicated term with id: #{term_id}, #{file_path}"
+        log "FATAL: Duplicated term with id: #{term_id}, #{fpath}"
       end
       encountered_terms[term_id] = true
     end
 
+    # Assume that definition is located in clause 3 of the ISO document
+    # in order. We really don't have a good reference here.
+    ref_clause = "3.#{index}"
+
     concept = Stepmod::Utils::Concept.parse(
       definition,
       reference_anchor: bibdata.anchor,
-      # Assume that definition is located in clause 3 of the ISO document
-      # in order. We really don't have a good reference here.
-      reference_clause: "3.#{index}"
+      reference_clause: ref_clause,
+      file_path: fpath + revision_string
     )
 
     parsed_terms << concept
     parsed_bibliography << bibdata
   end
 
+  log "INFO: Completed processing XML file #{fpath}"
+
 end
 
 parsed_bibliography.uniq!
@@ -98,10 +141,10 @@ File.open('031-generated-terms.adoc', 'w') { |file|
   file.puts(parsed_terms.map(&:to_mn_adoc).join("\n"))
 }
 
-log "written to: 031-generated-terms.adoc"
+log "INFO: written to: 031-generated-terms.adoc"
 
 File.open('991-generated-bibliography.adoc', 'w') { |file|
   file.puts(parsed_bibliography.map(&:to_mn_adoc).join("\n"))
 }
 
-log "written to: 991-generated-bibliography.adoc"
+log "INFO: written to: 991-generated-bibliography.adoc"