stepmod-utils 0.2.7 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe9caf0ee63e637eb1a73289d7394a495b5f8a313c6ee3d69aff2bdcc150e57f
-  data.tar.gz: 791c199203956d117839a900869fd61362ead66703f6723cd4b5c0c8d871ccc4
+  metadata.gz: e4b01b98d2e819f59adda9b7583eb65e0f3f17c67b57e297d4c63a28587a9c81
+  data.tar.gz: 274c0b90422ffb4f51134c49c4b1f8b9b8d4319b4369fd26baccf6191cf11183
 SHA512:
-  metadata.gz: 1910feb532333a698daa864f4387ccefdbcdeb6ccbcf3a962d3e7ec5dfa0a0c0f17643e8eba22ed625aa2a308595b0c30ea7d49ce5bed2de0a8c638cb1e2e00d
-  data.tar.gz: 19757106a503652a64052eb823750bb5a618750b68bf42abdc3833dde6be9d3345dbc80c5051d37cea2507fe07c87b0669cf3af60c66cf6579ace1cd02f9bf96
+  metadata.gz: 74483e26bbfdbb7dd5827ebf8d1b7270b13cf35765107830a7d5ce29239d4fa865804e6384ba092f7209fdc7ab91387f8841791822deb6da45d97f37d1621e82
+  data.tar.gz: 9d130371dc91aa996896c79c3d3a7ff4be703d258030e9d2aee64212add70b79f8fdc9962c9d3d2d1d309d8e5c771817a22f44ff515f3702c014c859c9013dd0

data/.github/workflows/rake.yml

@@ -16,19 +16,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
+        ruby: [ '3.0', '2.7', '2.6', '2.5', '2.4' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
-        include:
-          - ruby: '3.0'
-            os: 'ubuntu-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'windows-latest'
-            experimental: true
-          - ruby: '3.0'
-            os: 'macos-latest'
-            experimental: true
     steps:
       - uses: actions/checkout@v2
         with:

data/.github/workflows/release.yml

@@ -9,7 +9,7 @@ on:
         description: |
           Next release version. Possible values: x.y.z, major, minor, patch or pre|rc|etc
         required: true
-        default: 'patch'
+        default: 'skip'
   push:
     tags: [ v* ]
 
@@ -22,10 +22,11 @@ jobs:
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: '2.6'
+          bundler-cache: true
 
       - run: gem install gem-release
 
-      - if: ${{ github.event_name == 'workflow_dispatch' }}
+      - if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.next_version != 'skip' }}
         run: |
           git config user.name github-actions
           git config user.email github-actions@github.com
@@ -36,7 +37,7 @@ jobs:
           RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}
         run: |
           gem install gem-release
-          cat > ~/.gem/credentials << EOF
+          envsubst << 'EOF' > ~/.gem/credentials
           ---
           :rubygems_api_key: ${RUBYGEMS_API_KEY}
           EOF

data/.hound.yml

@@ -0,0 +1,5 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+ruby:
+  enabled: true
+  config_file: .rubocop.yml

data/.rubocop.yml

@@ -0,0 +1,10 @@
+# Auto-generated by Cimas: Do not edit it manually!
+# See https://github.com/metanorma/cimas
+inherit_from:
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+
+# local repo-specific modifications
+# ...
+
+AllCops:
+  TargetRubyVersion: 2.4

data/exe/stepmod-extract-terms

@@ -17,134 +17,103 @@ class Gem::Specification
 end
 
 require 'bundler/setup'
-require 'stepmod/utils/stepmod_definition_converter'
-require 'stepmod/utils/bibdata'
-require 'stepmod/utils/concept'
-require 'ptools'
-
-ReverseAdoc.config.unknown_tags = :bypass
-
-parsed_terms = []
-parsed_bibliography = []
-encountered_terms = {}
+require 'stepmod/utils/terms_extractor'
 
 stepmod_dir = ARGV.first || Dir.pwd
 
+general_concepts,
+resource_concepts,
+parsed_bibliography,
+part_concepts,
+part_resources,
+part_modules = Stepmod::Utils::TermsExtractor.call(stepmod_dir)
+
+def part_to_title(bibdata)
+  case bibdata.part.to_i
+  when 41
+    "Part 41"
+  when 42
+    "Geometric and topological representation"
+  when 43
+    "Foundation representation"
+  when 44
+    "Product structure, concept and configuration"
+  when 45
+    "Part 45"
+  when 46
+    "Visual presentation"
+  when 47
+    "Shape tolerance"
+  when 51
+    "Mathematical representation"
+  else
+    bibdata.title_en
+  end
+end
+
 def log message
   puts "[stepmod-utils] #{message}"
 end
 
-stepmod_path = Pathname.new(stepmod_dir).realpath
-
-# If we are using the stepmod CVS repository, provide the revision number per file
-has_cvs = File.which("cvs")
-cvs_mode = has_cvs && Dir.exists?(stepmod_path.join('CVS'))
-
-log "INFO: STEPmod directory set to #{stepmod_dir}."
-
-if cvs_mode
-  log "INFO: STEPmod directory is a CVS repository and will detect revisions."
-  log "INFO: [CVS] Detecting file revisions can be slow, please be patient!"
-else
-  log "INFO: STEPmod directory is not a CVS repository, skipping revision detection."
+part_concepts.each do |(bibdata, current_part_concepts)|
+  fn = "03x-stepmod-#{bibdata.part}.adoc"
+  File.open(fn, 'w') { |file|
+    file.puts("== #{part_to_title(bibdata)}\n\n")
+    file.puts(current_part_concepts.map(&:to_mn_adoc).join("\n"))
+  }
+  log "INFO: written to: #{fn}"
 end
 
-log "INFO: Detecting paths..."
-
-files = %w(
-  resource.xml
-  application_protocol.xml
-  business_object_model.xml
-  module.xml
-  ).inject([]) do |acc, t|
-    acc << Dir["#{stepmod_dir}/**/#{t}"]
-end.flatten.sort.uniq
-
-files.each do |file_path|
-  file_path = Pathname.new(file_path).realpath
-  fpath = file_path.relative_path_from(stepmod_path)
-
-  log "INFO: Processing XML file #{fpath}"
-  current_document = Nokogiri::XML(File.read(file_path)).root
-
-  bibdata = nil
-  begin
-    bibdata = Stepmod::Utils::Bibdata.new(document: current_document)
-  rescue
-    log "WARNING: Unknown file #{fpath}, skipped"
-    next
-  end
-
-  # TODO: we may want a command line option to override this in the future
-  unless %w(IS DIS FDIS TS).include? bibdata.doctype
-    log "INFO: skipped #{bibdata.docid} as it is not IS, DIS or TS"
-    next
-  end
 
-  if bibdata.part.to_s.empty?
-    log "FATAL: missing `part` attribute: #{fpath}"
-    log "INFO: skipped #{bibdata.docid} as it is missing `part` attribute."
-    next
-  end
+part_resources.each do |(bibdata, current_part_resources)|
+  fn = "04x-stepmod-entities-resources-#{bibdata.part}.adoc"
+  File.open(fn, 'w') { |file|
+    file.puts("== #{part_to_title(bibdata)}\n\n")
+    file.puts(current_part_resources.map(&:to_mn_adoc).join("\n"))
+  }
+  log "INFO: written to: #{fn}"
+end
 
-  revision_string = "\n// CVS: revision not detected"
-  if cvs_mode
-    # Run `cvs status` to find out version
-
-    log "INFO: Detecting CVS revision..."
-    Dir.chdir(stepmod_path) do
-      status = `cvs status #{fpath}`
-
-      unless status.empty?
-        working_rev = status.split(/\n/).grep(/Working revision:/).first.match(/revision:\s+(.+)$/)[1]
-        repo_rev = status.split(/\n/).grep(/Repository revision:/).first.match(/revision:\t(.+)\t/)[1]
-        log "INFO: CVS working rev (#{working_rev}), repo rev (#{repo_rev})"
-        revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
-          "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
-      end
-    end
-  end
 
-  # read definitions
-  current_document.xpath('//definition').each.with_index(1) do |definition, index|
+part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
+  fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
+  File.open(fn, 'w') { |file|
+    file.puts("")
+    unless part_modules_arm.empty?
+      schema_name = part_modules_arm.first.first
+      concepts = part_modules_arm.first.last
 
-    term_id = definition['id']
-    unless term_id.nil?
-      if encountered_terms[term_id]
-        log "FATAL: Duplicated term with id: #{term_id}, #{fpath}"
-      end
-      encountered_terms[term_id] = true
+      # puts "SCHEMA NAME ARM: #{schema_name}"
+      file.puts("== #{schema_name}\n\n")
+      file.puts(concepts.map(&:to_mn_adoc).join("\n"))
     end
 
-    # Assume that definition is located in clause 3 of the ISO document
-    # in order. We really don't have a good reference here.
-    ref_clause = "3.#{index}"
-
-    concept = Stepmod::Utils::Concept.parse(
-      definition,
-      reference_anchor: bibdata.anchor,
-      reference_clause: ref_clause,
-      file_path: fpath + revision_string
-    )
+    file.puts("")
 
-    parsed_terms << concept
-    parsed_bibliography << bibdata
-  end
+    unless part_modules_mim.empty?
+      schema_name = part_modules_mim.first.first
 
-  log "INFO: Completed processing XML file #{fpath}"
+      # puts "SCHEMA NAME MIM: #{schema_name}"
+      concepts = part_modules_mim.first.last
+      file.puts("== #{schema_name}\n\n")
+      file.puts(concepts.map(&:to_mn_adoc).join("\n"))
+    end
 
+  }
+  log "INFO: written to: #{fn}"
 end
 
-parsed_bibliography
-
-File.open('031-generated-terms.adoc', 'w') { |file|
-  file.puts(parsed_terms.map(&:to_mn_adoc).join("\n"))
+File.open('031-stepmod-general.adoc', 'w') { |file|
+  file.puts(general_concepts.map(&:to_mn_adoc).join("\n"))
 }
+log "INFO: written to: 031-stepmod-general.adoc"
 
-log "INFO: written to: 031-generated-terms.adoc"
+File.open('041-stepmod-entities-resources.adoc', 'w') { |file|
+  file.puts(resource_concepts.map(&:to_mn_adoc).join("\n"))
+}
+log "INFO: written to: 041-stepmod-entities-resources.adoc"
 
 File.open('991-generated-bibliography.adoc', 'w') { |file|
-  file.puts(parsed_bibliography.map(&:to_mn_adoc).uniq.join("\n"))
+  file.puts(parsed_bibliography.map(&:to_mn_adoc).sort.uniq.join("\n"))
 }
-
 log "INFO: written to: 991-generated-bibliography.adoc"

data/lib/stepmod/utils/bibdata.rb

@@ -67,6 +67,8 @@ module Stepmod
           "ISO/CD #{DOCNUMBER}-#{part}"
         when "DIS"
           "ISO/DIS #{DOCNUMBER}-#{part}"
+        when "FDIS"
+          "ISO/FDIS #{DOCNUMBER}-#{part}"
         when "TS"
           "ISO/TS #{DOCNUMBER}-#{part}"
         when "CD-TS"

data/lib/stepmod/utils/concept.rb

@@ -20,22 +20,96 @@ module Stepmod
         end
       end
 
-      def self.parse(definition_xml, reference_anchor:, reference_clause:, file_path:)
-        new(
-          converted_definition: Stepmod::Utils::StepmodDefinitionConverter.convert(definition_xml),
-          reference_anchor: reference_anchor,
-          reference_clause: reference_clause,
-          file_path: file_path
-        )
+      class << self
+        def parse(definition_xml, reference_anchor:, reference_clause:, file_path:)
+          converted_definition = Stepmod::Utils::StepmodDefinitionConverter.convert(
+            definition_xml,
+            {
+              # We don't want examples and notes
+              no_notes_examples: true,
+              reference_anchor: reference_anchor
+            }
+          )
+
+          return nil if converted_definition.nil? || converted_definition.strip.empty?
+
+          if definition_xml.name == 'ext_description'
+            converted_definition = <<~TEXT
+              #{converted_definition}
+
+              NOTE: This term is incompletely defined in this document.
+              Reference <<#{reference_anchor}>> for the complete definition.
+            TEXT
+          end
+          # https://github.com/metanorma/stepmod-utils/issues/86
+          if definition_xml.name == 'definition'
+            designation = definition_designation(definition_xml)
+            definition = definition_xml_definition(definition_xml, reference_anchor)
+            converted_definition = definition_xml_converted_definition(designation, definition).strip
+          end
+          new(
+            designation: designation,
+            definition: definition,
+            converted_definition: converted_definition,
+            reference_anchor: reference_anchor,
+            reference_clause: reference_clause,
+            file_path: file_path
+          )
+        end
+
+        def definition_designation(definition_xml)
+          alts = definition_xml.xpath('.//def/p').map(&:text)
+          {
+            accepted: definition_xml.xpath('.//term').first&.text,
+            alt: alts
+          }
+        end
+
+        def definition_xml_definition(definition_xml, reference_anchor)
+          text_nodes = definition_xml
+                        .xpath('.//def')
+                        .first
+                        .children
+                        .reject { |n| n.name == 'p'  }
+          wrapper = "<def>#{text_nodes.map(&:to_s).join}</def>"
+          Stepmod::Utils::Converters::Def
+            .new
+            .convert(
+              Nokogiri::XML(wrapper).root,
+              {
+                # We don't want examples and notes
+                no_notes_examples: true,
+                reference_anchor: reference_anchor
+              })
+        end
+
+        def definition_xml_converted_definition(designation, definition)
+          if designation[:alt].length.positive?
+            alt_notation = "alt:[#{designation[:alt].map(&:strip).join(',')}]"
+          end
+          result = <<~TEXT
+            === #{designation[:accepted]}
+          TEXT
+          if alt_notation
+            result += <<~TEXT
+
+              #{alt_notation}
+            TEXT
+          end
+          <<~TEXT
+            #{result}
+            #{definition}
+          TEXT
+        end
       end
 
       def to_mn_adoc
         <<~TEXT
-          // STEPmod path: #{file_path}
+          // STEPmod path:#{!file_path.empty? ? " #{file_path}" : ""}
           #{converted_definition}
 
           [.source]
-          <<#{reference_anchor},clause=#{reference_clause}>>
+          <<#{reference_anchor}#{reference_clause ? ",clause=" + reference_clause : ""}>>
 
         TEXT
       end

data/lib/stepmod/utils/converters/arm.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class Arm < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          treat_children(node, state)
+        end
+
+        private
+
+        def treat_children(node, state)
+          res = node.children.map { |child| treat(child, state) }
+          res.map(&:strip).reject(&:empty?).join("\n\n")
+        end
+      end
+
+      ReverseAdoc::Converters.register :arm, Arm.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/def.rb

@@ -12,13 +12,15 @@ module Stepmod
 
         def treat_children(node, state)
           converted = node.children.each_with_object({}) do |child, res|
-                        content = treat(child, state).strip
-                        next if content.empty?
 
-                        res[child] = content
-                      end
+            content = treat(child, state)
+            next if content.strip.empty?
+
+            res[child] = content
+          end
           previous = nil
           result = ''
+
           converted.each.with_index do |(child, content), i|
             if block_tag?(child, previous)
               result += "\n\n"
@@ -30,6 +32,11 @@ module Stepmod
             result += content
             previous = child
           end
+
+          # Remove double newlines for every line
+          result = result.gsub(/\n\n+/, "\n\n")
+          result = result.squeeze(' ')
+
           result.strip
         end
 
@@ -50,8 +57,9 @@ module Stepmod
           return unless can_transform_to_alt?(first_child_tag)
 
           result = Stepmod::Utils::Converters::Synonym
-                    .new
-                    .convert(first_child_tag)
+            .new
+            .convert(first_child_tag)
+
           first_child_tag.remove
           "#{result}\n\n"
         end
@@ -71,6 +79,7 @@ module Stepmod
       end
 
       ReverseAdoc::Converters.register :def, Def.new
+      ReverseAdoc::Converters.register :description, Def.new
     end
   end
 end

data/lib/stepmod/utils/converters/example.rb

@@ -5,10 +5,21 @@ module Stepmod
     module Converters
       class Example < ReverseAdoc::Converters::Base
         def convert(node, state = {})
-          "\n\n[example]\n====\n#{treat_children(node, state).strip}\n====\n\n"
+
+          # If we want to skip this node
+          return '' if state[:no_notes_examples]
+
+          <<~TEMPLATE
+
+          [example]
+          ====
+          #{treat_children(node, state).strip}
+          ====
+
+          TEMPLATE
         end
       end
       ReverseAdoc::Converters.register :example, Example.new
     end
   end
-end
+end

data/lib/stepmod/utils/converters/ext_description.rb

@@ -4,9 +4,11 @@ module Stepmod
       class ExtDescription < ReverseAdoc::Converters::Base
         def convert(node, state = {})
           state = state.merge(schema_name: node['linkend'])
+          child_text = treat_children(node, state).strip
+
           <<~TEMPLATE
             (*"#{node['linkend']}"
-            #{treat_children(node, state).strip}
+            #{child_text}
             *)
           TEMPLATE
         end

data/lib/stepmod/utils/converters/figure.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+require 'reverse_adoc/converters/figure'
+
+module Stepmod
+  module Utils
+    module Converters
+      class Figure < ReverseAdoc::Converters::Figure
+        def convert(node, state = {})
+          # If we want to skip this node
+          return '' if state[:no_notes_examples]
+
+          super
+        end
+      end
+      # This replaces the converter
+      ReverseAdoc::Converters.register :figure, Figure.new
+    end
+  end
+end
+

data/lib/stepmod/utils/converters/module_ref.rb

@@ -13,10 +13,14 @@ module Stepmod
           # We take the text value of the element and convert to this:
 
           # term:[individual products]
-          if node['linkend'].split(':').length > 1
-            ref = node.text
+
+          ref = node.text.strip
+          if !ref.empty?
+            " term:[#{normalized_ref(ref)}] "
+          elsif
+            ref = node['linkend'].split(':').first
+            " *#{ref}*"
           end
-          " term:[#{normalized_ref(ref)}] "
         end
 
         private

data/lib/stepmod/utils/converters/note.rb

@@ -5,10 +5,21 @@ module Stepmod
     module Converters
       class Note < ReverseAdoc::Converters::Base
         def convert(node, state = {})
-          "\n\n[NOTE]\n--\n#{treat_children(node, state).strip}\n--\n\n"
+
+          # If we want to skip this node
+          return '' if state[:no_notes_examples]
+
+          <<~TEMPLATE
+
+          [NOTE]
+          --
+          #{treat_children(node, state).strip}
+          --
+
+          TEMPLATE
         end
       end
       ReverseAdoc::Converters.register :note, Note.new
     end
   end
-end
+end

data/lib/stepmod/utils/converters/stepmod_ext_description.rb

@@ -0,0 +1,44 @@
+module Stepmod
+  module Utils
+    module Converters
+      class StepmodExtDescription < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          state = state.merge(schema_name: node['linkend'])
+          linkend = node['linkend'].split('.')
+
+          # We only want ENTITY entries, not their attributes
+          # https://github.com/metanorma/iso-10303-2/issues/36#issuecomment-841300092
+          return nil if linkend.length != 2
+
+          child_text = treat_children(node, state).strip
+          return nil if child_text.empty?
+
+          # Only taking the first paragraph of the definition
+          child_text = child_text.split("\n").first
+
+          # # Only taking the first sentence
+          # if child_text.contains?(".")
+          #   child_text = child_text.split(".").first
+          # end
+
+          domain =  case linkend.first
+                    when /_mim$/, /_arm$/
+                      "STEP module"
+                    # when /_schema$/
+                    else
+                      "STEP resource"
+                    end
+
+          <<~TEMPLATE
+            === #{linkend.last}
+
+            #{domain ? "domain:[" + domain + "]" : ""}
+
+            #{child_text}
+          TEMPLATE
+        end
+      end
+      ReverseAdoc::Converters.register :ext_description, StepmodExtDescription.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/text.rb

@@ -30,8 +30,8 @@ module Stepmod
         def treat_text(node)
           text = node.text
           text = preserve_nbsp(text)
-          # text = remove_border_newlines(text)
           text = remove_inner_newlines(text)
+          text = remove_border_newlines(text)
 
           text = preserve_keychars_within_backticks(text)
           text = preserve_tags(text)

data/lib/stepmod/utils/converters/uof.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'stepmod/utils/converters/synonym'
+
+module Stepmod
+  module Utils
+    module Converters
+      class Uof < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+
+          # WARNING: <uof> tag content is deprecated
+          return ""
+
+          #
+          # <<~TEXT
+          # === #{node['name'].strip}
+
+          # <STEP module> #{treat_children(node, state).strip}
+          # TEXT
+        end
+      end
+
+      ReverseAdoc::Converters.register :uof, Uof.new
+    end
+  end
+end

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -64,4 +64,4 @@ module Stepmod
       end
     end
   end
-end
+end

data/lib/stepmod/utils/stepmod_definition_converter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'reverse_adoc'
+require 'stepmod/utils/converters/arm'
 require 'stepmod/utils/converters/clause_ref'
 require 'stepmod/utils/converters/express_ref'
 require 'stepmod/utils/converters/module_ref'
@@ -11,8 +12,11 @@ require 'stepmod/utils/converters/example'
 require 'stepmod/utils/converters/note'
 require 'stepmod/utils/converters/ol'
 require 'stepmod/utils/converters/stem'
+require 'stepmod/utils/converters/stepmod_ext_description'
 require 'stepmod/utils/converters/term'
 require 'stepmod/utils/converters/synonym'
+require 'stepmod/utils/converters/uof'
+require 'stepmod/utils/converters/figure'
 
 require 'reverse_adoc/converters/a'
 require 'reverse_adoc/converters/blockquote'
@@ -37,19 +41,21 @@ module Stepmod
   module Utils
     class StepmodDefinitionConverter
       def self.convert(input, options = {})
-        root = if input.is_a?(String)
-                  then Nokogiri::XML(input).root
-                elsif input.is_a?(Nokogiri::XML::Document)
-                  then input.root
-                elsif input.is_a?(Nokogiri::XML::Node)
-                  then input
-                end
+        root = case input
+               when String
+                 Nokogiri::XML(input).root
+               when Nokogiri::XML::Document
+                 input.root
+               when Nokogiri::XML::Node
+                 input
+               end
 
-        root || (return '')
+        return '' unless root
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
-          ReverseAdoc.cleaner.tidy(result)
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
+          return '' unless result
+          ReverseAdoc.cleaner.tidy(result.dup)
         end
       end
     end

data/lib/stepmod/utils/terms_extractor.rb

@@ -0,0 +1,326 @@
+require 'stepmod/utils/stepmod_definition_converter'
+require 'stepmod/utils/bibdata'
+require 'stepmod/utils/concept'
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+module Stepmod
+  module Utils
+    class TermsExtractor
+      # TODO: we may want a command line option to override this in the future
+      ACCEPTED_STAGES = %w(IS DIS FDIS TS)
+
+      attr_reader :stepmod_path,
+        :stepmod_dir,
+        :general_concepts,
+        :resource_concepts,
+        :parsed_bibliography,
+        :encountered_terms,
+        :cvs_mode,
+        :part_concepts,
+        :part_resources,
+        :part_modules,
+        :stdout
+
+      def self.call(stepmod_dir, stdout = STDOUT)
+        new(stepmod_dir, stdout).call
+      end
+
+      def initialize(stepmod_dir, stdout)
+        @stdout = stdout
+        @stepmod_dir = stepmod_dir
+        @stepmod_path = Pathname.new(stepmod_dir).realpath
+        @general_concepts = []
+        @resource_concepts = []
+        @parsed_bibliography = []
+        @part_concepts = []
+        @part_resources = []
+        @part_modules = []
+        @encountered_terms = {}
+      end
+
+      def log message
+        stdout.puts "[stepmod-utils] #{message}"
+      end
+
+      def term_special_category(bibdata)
+        case bibdata.part.to_i
+        when 41,42,43,44,45,46,47,51
+          true
+        when [56..112]
+          true
+        else
+          false
+        end
+      end
+
+      def call
+        # If we are using the stepmod CVS repository, provide the revision number per file
+        @cvs_mode = if Dir.exists?(stepmod_path.join('CVS'))
+          require 'ptools'
+          # ptools provides File.which
+          File.which("cvs")
+        end
+
+        log "INFO: STEPmod directory set to #{stepmod_dir}."
+
+        if cvs_mode
+          log "INFO: STEPmod directory is a CVS repository and will detect revisions."
+          log "INFO: [CVS] Detecting file revisions can be slow, please be patient!"
+        else
+          log "INFO: STEPmod directory is not a CVS repository, skipping revision detection."
+        end
+
+        log "INFO: Detecting paths..."
+
+        repo_index = Nokogiri::XML(File.read(stepmod_path.join('repository_index.xml'))).root
+
+        files = []
+
+        # add module paths
+        repo_index.xpath('//module').each do |x|
+          path = Pathname.new("#{stepmod_dir}/modules/#{x['name']}/module.xml")
+          files << path if File.exists? path
+        end
+
+        # add resource_docs paths
+        repo_index.xpath('//resource_doc').each do |x|
+          path = Pathname.new("#{stepmod_dir}/resource_docs/#{x['name']}/resource.xml")
+          files << path if File.exists? path
+        end
+
+        # add business_object_models paths
+        repo_index.xpath('//business_object_model').each do |x|
+          path = Pathname.new("#{stepmod_dir}/business_object_models/#{x['name']}/business_object_model.xml")
+          files << path if File.exists? path
+        end
+
+        # add application_protocols paths
+        repo_index.xpath('//application_protocol').each do |x|
+          path = Pathname.new("#{stepmod_dir}/application_protocols/#{x['name']}/application_protocol.xml")
+          files << path if File.exists? path
+        end
+
+        files.sort!.uniq!
+        process_term_files(files)
+
+        [
+          general_concepts,
+          resource_concepts,
+          parsed_bibliography,
+          part_concepts,
+          part_resources,
+          part_modules
+        ]
+      end
+
+      private
+
+      def process_term_files(files)
+        parsed_schema_names = {}
+        files.each do |file_path|
+          file_path = file_path.realpath
+          fpath = file_path.relative_path_from(stepmod_path)
+
+          log "INFO: Processing XML file #{fpath}"
+          current_document = Nokogiri::XML(File.read(file_path)).root
+
+          bibdata = nil
+          begin
+            bibdata = Stepmod::Utils::Bibdata.new(document: current_document)
+          rescue
+            log "WARNING: Unknown file #{fpath}, skipped"
+            next
+          end
+
+          unless ACCEPTED_STAGES.include? bibdata.doctype
+            log "INFO: skipped #{bibdata.docid} as it is not one of (#{ACCEPTED_STAGES.join(", ")})."
+            next
+          end
+
+          if bibdata.part.to_s.empty?
+            log "FATAL: missing `part` attribute: #{fpath}"
+            log "INFO: skipped #{bibdata.docid} as it is missing `part` attribute."
+            next
+          end
+
+          revision_string = "\n// CVS: revision not detected"
+          if cvs_mode
+            # Run `cvs status` to find out version
+
+            log "INFO: Detecting CVS revision..."
+            Dir.chdir(stepmod_path) do
+              status = `cvs status #{fpath}`
+
+              unless status.empty?
+                working_rev = status.split(/\n/).grep(/Working revision:/).first.match(/revision:\s+(.+)$/)[1]
+                repo_rev = status.split(/\n/).grep(/Repository revision:/).first.match(/revision:\t(.+)\t/)[1]
+                log "INFO: CVS working rev (#{working_rev}), repo rev (#{repo_rev})"
+                revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
+                  "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
+              end
+            end
+          end
+
+          # read definitions
+          current_part_concepts = []
+          definition_index = 0
+          current_document.xpath('//definition').each do |definition|
+            definition_index += 1
+            term_id = definition['id']
+            unless term_id.nil?
+              if encountered_terms[term_id]
+                log "FATAL: Duplicated term with id: #{term_id}, #{fpath}"
+              end
+              encountered_terms[term_id] = true
+            end
+
+            # Assume that definition is located in clause 3 of the ISO document
+            # in order. We really don't have a good reference here.
+            ref_clause = "3.#{definition_index}"
+
+            concept = Stepmod::Utils::Concept.parse(
+              definition,
+              reference_anchor: bibdata.anchor,
+              reference_clause: ref_clause,
+              file_path: fpath + revision_string
+            )
+            next unless concept
+
+            unless term_special_category(bibdata)
+              # log "INFO: this part is generic"
+              general_concepts << concept
+            else
+              # log "INFO: this part is special"
+              current_part_concepts << concept
+            end
+
+            parsed_bibliography << bibdata
+          end
+
+          current_part_resources = []
+          current_part_modules_arm = {}
+          current_part_modules_mim = {}
+
+          log "INFO: FILE PATH IS #{file_path}"
+          case file_path.to_s
+          when /resource.xml$/
+            log "INFO: Processing resource.xml for #{file_path}"
+            # Assumption: every schema is only linked by a single resource_docs document.
+            current_document.xpath('//schema').each do |schema_node|
+              schema_name = schema_node['name']
+              if parsed_schema_names[schema_name]
+                log "ERROR: We have encountered this schema before: #{schema_name} from path #{parsed_schema_names[schema_name]}, now at #{file_path}"
+                next
+              else
+                parsed_schema_names[schema_name] = file_path
+              end
+
+              Dir["#{stepmod_path}/resources/#{schema_name}/descriptions.xml"].each do |description_xml_path|
+                log "INFO: Processing resources schema #{description_xml_path}"
+                description_document = Nokogiri::XML(File.read(description_xml_path)).root
+                description_document.xpath('//ext_description').each do |ext_description|
+
+                  # log "INFO: Processing linkend[#{ext_description['linkend']}]"
+
+                  concept = Stepmod::Utils::Concept.parse(
+                    ext_description,
+                    reference_anchor: bibdata.anchor,
+                    reference_clause: nil,
+                    file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                  )
+                  next unless concept
+
+                  unless term_special_category(bibdata)
+                    # log "INFO: this part is generic"
+                    resource_concepts << concept
+                  else
+                    # log "INFO: this part is special"
+                    current_part_resources << concept
+                  end
+
+                  parsed_bibliography << bibdata
+                end
+              end
+            end
+
+          when /module.xml$/
+            log "INFO: Processing module.xml for #{file_path}"
+            # Assumption: every schema is only linked by a single module document.
+            # puts current_document.xpath('//module').length
+            schema_name = current_document.xpath('//module').first['name']
+            if parsed_schema_names[schema_name]
+              log "ERROR: We have encountered this schema before: #{schema_name} from path #{parsed_schema_names[schema_name]}, now at #{file_path}"
+              next
+            else
+              parsed_schema_names[schema_name] = file_path
+            end
+
+            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/arm_descriptions.xml"
+            log "INFO: Processing modules schema #{description_xml_path}"
+
+            if File.exists?(description_xml_path)
+              description_document = Nokogiri::XML(File.read(description_xml_path)).root
+              description_document.xpath('//ext_description').each do |ext_description|
+
+                linkend_schema = ext_description['linkend'].split('.').first
+                concept = Stepmod::Utils::Concept.parse(
+                  ext_description,
+                  reference_anchor: bibdata.anchor,
+                  reference_clause: nil,
+                  file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                )
+                next unless concept
+
+                current_part_modules_arm[linkend_schema] ||= []
+                current_part_modules_arm[linkend_schema] << concept
+                # puts part_modules_arm.inspect
+                parsed_bibliography << bibdata
+              end
+            end
+
+            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/mim_descriptions.xml"
+            log "INFO: Processing modules schema #{description_xml_path}"
+
+            if File.exists?(description_xml_path)
+              description_document = Nokogiri::XML(File.read(description_xml_path)).root
+              description_document.xpath('//ext_description').each do |ext_description|
+
+                linkend_schema = ext_description['linkend'].split('.').first
+
+                concept = Stepmod::Utils::Concept.parse(
+                  ext_description,
+                  reference_anchor: bibdata.anchor,
+                  reference_clause: nil,
+                  file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                )
+                next unless concept
+
+                current_part_modules_mim[linkend_schema] ||= []
+                current_part_modules_mim[linkend_schema] << concept
+
+                parsed_bibliography << bibdata
+              end
+            end
+
+          end
+
+          log "INFO: Completed processing XML file #{fpath}"
+          if current_part_concepts.empty?
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) because it contains no concepts."
+          elsif current_part_concepts.length < 3
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) because it only has #{current_part_concepts.length} terms."
+
+            current_part_concepts.each do |x|
+              general_concepts << x
+            end
+          else
+            part_concepts << [bibdata, current_part_concepts] unless current_part_concepts.empty?
+          end
+          part_resources << [bibdata, current_part_resources] unless current_part_resources.empty?
+          part_modules << [bibdata, current_part_modules_arm, current_part_modules_mim] if current_part_modules_arm.size + current_part_modules_mim.size > 0
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.2.7"
+    VERSION = "0.3.5"
   end
 end

data/stepmod-utils.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "thor", ">= 0.20.3"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.2.9"
-  spec.add_runtime_dependency "ptools", '~> 1.3'
   spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "ptools"
   spec.add_development_dependency "byebug", "~> 11.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.5
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -39,21 +39,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.2.9
 - !ruby/object:Gem::Dependency
-  name: ptools
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: ptools
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -98,7 +98,9 @@ files:
 - ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
 - ".gitignore"
+- ".hound.yml"
 - ".rspec"
+- ".rubocop.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Makefile
@@ -117,6 +119,7 @@ files:
 - lib/stepmod/utils/cleaner.rb
 - lib/stepmod/utils/concept.rb
 - lib/stepmod/utils/converters/a.rb
+- lib/stepmod/utils/converters/arm.rb
 - lib/stepmod/utils/converters/blockquote.rb
 - lib/stepmod/utils/converters/br.rb
 - lib/stepmod/utils/converters/bypass.rb
@@ -138,6 +141,7 @@ files:
 - lib/stepmod/utils/converters/express_ref_express_description.rb
 - lib/stepmod/utils/converters/ext_description.rb
 - lib/stepmod/utils/converters/ext_descriptions.rb
+- lib/stepmod/utils/converters/figure.rb
 - lib/stepmod/utils/converters/fund_cons.rb
 - lib/stepmod/utils/converters/head.rb
 - lib/stepmod/utils/converters/hr.rb
@@ -153,6 +157,7 @@ files:
 - lib/stepmod/utils/converters/schema.rb
 - lib/stepmod/utils/converters/schema_diag.rb
 - lib/stepmod/utils/converters/stem.rb
+- lib/stepmod/utils/converters/stepmod_ext_description.rb
 - lib/stepmod/utils/converters/strong.rb
 - lib/stepmod/utils/converters/sub.rb
 - lib/stepmod/utils/converters/sup.rb
@@ -160,11 +165,13 @@ files:
 - lib/stepmod/utils/converters/table.rb
 - lib/stepmod/utils/converters/term.rb
 - lib/stepmod/utils/converters/text.rb
+- lib/stepmod/utils/converters/uof.rb
 - lib/stepmod/utils/html_to_asciimath.rb
 - lib/stepmod/utils/smrl_description_converter.rb
 - lib/stepmod/utils/smrl_resource_converter.rb
 - lib/stepmod/utils/stepmod_definition_converter.rb
 - lib/stepmod/utils/stepmod_file_annotator.rb
+- lib/stepmod/utils/terms_extractor.rb
 - lib/stepmod/utils/version.rb
 - migrating_from_cvs.adoc
 - resource_example.xml
@@ -188,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.