stepmod-utils 0.3.2 → 0.3.8

data/lib/stepmod/utils/stepmod_definition_converter.rb

@@ -1,40 +1,40 @@
 # frozen_string_literal: true
 
-require 'reverse_adoc'
-require 'stepmod/utils/converters/arm'
-require 'stepmod/utils/converters/clause_ref'
-require 'stepmod/utils/converters/express_ref'
-require 'stepmod/utils/converters/module_ref'
-require 'stepmod/utils/converters/def'
-require 'stepmod/utils/converters/definition'
-require 'stepmod/utils/converters/em'
-require 'stepmod/utils/converters/example'
-require 'stepmod/utils/converters/note'
-require 'stepmod/utils/converters/ol'
-require 'stepmod/utils/converters/stem'
-require 'stepmod/utils/converters/stepmod_ext_description'
-require 'stepmod/utils/converters/term'
-require 'stepmod/utils/converters/synonym'
-require 'stepmod/utils/converters/uof'
-
-require 'reverse_adoc/converters/a'
-require 'reverse_adoc/converters/blockquote'
-require 'reverse_adoc/converters/bypass'
-require 'reverse_adoc/converters/br'
-require 'reverse_adoc/converters/code'
-require 'reverse_adoc/converters/drop'
-require 'reverse_adoc/converters/head'
-require 'reverse_adoc/converters/hr'
-require 'reverse_adoc/converters/ignore'
-require 'reverse_adoc/converters/li'
-require 'reverse_adoc/converters/p'
-require 'reverse_adoc/converters/pass_through'
-require 'reverse_adoc/converters/q'
-require 'reverse_adoc/converters/strong'
-require 'reverse_adoc/converters/sup'
-require 'reverse_adoc/converters/sub'
-require 'reverse_adoc/converters/text'
+require "reverse_adoc"
+require "stepmod/utils/converters/arm"
+require "stepmod/utils/converters/clause_ref"
+require "stepmod/utils/converters/express_ref"
+require "stepmod/utils/converters/module_ref"
+require "stepmod/utils/converters/def"
+require "stepmod/utils/converters/definition"
+require "stepmod/utils/converters/em"
+require "stepmod/utils/converters/example"
+require "stepmod/utils/converters/note"
+require "stepmod/utils/converters/ol"
+require "stepmod/utils/converters/stem"
+require "stepmod/utils/converters/stepmod_ext_description"
+require "stepmod/utils/converters/term"
+require "stepmod/utils/converters/synonym"
+require "stepmod/utils/converters/uof"
+require "stepmod/utils/converters/figure"
 
+require "reverse_adoc/converters/a"
+require "reverse_adoc/converters/blockquote"
+require "reverse_adoc/converters/bypass"
+require "reverse_adoc/converters/br"
+require "reverse_adoc/converters/code"
+require "reverse_adoc/converters/drop"
+require "reverse_adoc/converters/head"
+require "reverse_adoc/converters/hr"
+require "reverse_adoc/converters/ignore"
+require "reverse_adoc/converters/li"
+require "reverse_adoc/converters/p"
+require "reverse_adoc/converters/pass_through"
+require "reverse_adoc/converters/q"
+require "reverse_adoc/converters/strong"
+require "reverse_adoc/converters/sup"
+require "reverse_adoc/converters/sub"
+require "reverse_adoc/converters/text"
 
 module Stepmod
   module Utils
@@ -49,11 +49,13 @@ module Stepmod
                  input
                end
 
-        return '' unless root
+        return "" unless root
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
-          return '' unless result
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root,
+                                                                     options)
+          return "" unless result
+
           ReverseAdoc.cleaner.tidy(result.dup)
         end
       end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -1,6 +1,6 @@
-require 'json'
-require 'stepmod/utils/smrl_description_converter'
-require 'stepmod/utils/smrl_resource_converter'
+require "json"
+require "stepmod/utils/smrl_description_converter"
+require "stepmod/utils/smrl_resource_converter"
 
 module Stepmod
   module Utils
@@ -17,35 +17,41 @@ module Stepmod
 
       def call
         match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
-        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : 'descriptions.xml'
-        descriptions_file = File.join(File.dirname(express_file), descriptions_base)
+        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : "descriptions.xml"
+        descriptions_file = File.join(File.dirname(express_file),
+                                      descriptions_base)
         output_express = File.read(express_file)
         resource_docs_cache = JSON.parse(File.read(resource_docs_cache_file))
 
         if File.exists?(descriptions_file)
           descriptions = Nokogiri::XML(File.read(descriptions_file)).root
           added_resource_descriptions = {}
-          descriptions.xpath('ext_description').each do |description|
+          descriptions.xpath("ext_description").each do |description|
             # Add base resource from linked path if exists, eg "language_schema.language.wr:WR1" -> "language_schema"
-            base_linked = description['linkend'].to_s.split('.').first
+            base_linked = description["linkend"].to_s.split(".").first
             if added_resource_descriptions[base_linked].nil?
-              base_reource_doc_dir = resource_docs_cache[description['linkend'].to_s.split('.').first]
+              base_reource_doc_dir = resource_docs_cache[description["linkend"].to_s.split(".").first]
               if base_reource_doc_dir
-                output_express << convert_from_resource_file(base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file)
+                output_express << convert_from_resource_file(
+                  base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file
+                )
               end
               added_resource_descriptions[base_linked] = true
             end
-            resource_docs_dir = resource_docs_cache[description['linkend']]
+            resource_docs_dir = resource_docs_cache[description["linkend"]]
             # Convert content description
             # when a schema description is available from resource.xml and also descriptions.xml, the description from resource.xml is only used.
             # https://github.com/metanorma/annotated-express/issues/32#issuecomment-792609078
             if description.text.strip.length.positive? && resource_docs_dir.nil?
-              output_express << convert_from_description_text(descriptions_file, description)
+              output_express << convert_from_description_text(
+                descriptions_file, description
+              )
             end
             # Add converted description from exact linked path
-            if resource_docs_dir && added_resource_descriptions[description['linkend']].nil?
-              output_express << convert_from_resource_file(resource_docs_dir, stepmod_dir, description['linkend'], descriptions_file)
-              added_resource_descriptions[description['linkend']] = true
+            if resource_docs_dir && added_resource_descriptions[description["linkend"]].nil?
+              output_express << convert_from_resource_file(resource_docs_dir,
+                                                           stepmod_dir, description["linkend"], descriptions_file)
+              added_resource_descriptions[description["linkend"]] = true
             end
           end
         end
@@ -57,20 +63,21 @@ module Stepmod
 
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
-          wrapper = "<ext_descriptions>#{description.to_s}</ext_descriptions>"
-          "\n" + Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)
+          wrapper = "<ext_descriptions>#{description}</ext_descriptions>"
+          "\n#{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)}"
         end
       end
 
       def convert_from_resource_file(resource_docs_dir, stepmod_dir, linked, descriptions_file)
-        resource_docs_file = File.join(stepmod_dir, 'data/resource_docs', resource_docs_dir, 'resource.xml')
+        resource_docs_file = File.join(stepmod_dir, "data/resource_docs",
+                                       resource_docs_dir, "resource.xml")
         puts(resource_docs_file)
         resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
         schema = resource_docs.xpath("schema[@name='#{linked}']")
 
         Dir.chdir(File.dirname(descriptions_file)) do
-          wrapper = "<resource>#{schema.to_s}</resource>"
-          "\n" + Stepmod::Utils::SmrlResourceConverter.convert(wrapper)
+          wrapper = "<resource>#{schema}</resource>"
+          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper)}"
         end
       end
     end

data/lib/stepmod/utils/terms_extractor.rb

@@ -0,0 +1,378 @@
+require "stepmod/utils/stepmod_definition_converter"
+require "stepmod/utils/bibdata"
+require "stepmod/utils/concept"
+require "glossarist"
+require "securerandom"
+
+ReverseAdoc.config.unknown_tags = :bypass
+
+module Stepmod
+  module Utils
+    class TermsExtractor
+      # TODO: we may want a command line option to override this in the future
+      ACCEPTED_STAGES = %w(IS DIS FDIS TS).freeze
+      WITHDRAWN_STATUS = "withdrawn".freeze
+
+      attr_reader :stepmod_path,
+                  :stepmod_dir,
+                  :general_concepts,
+                  :resource_concepts,
+                  :parsed_bibliography,
+                  :encountered_terms,
+                  :cvs_mode,
+                  :part_concepts,
+                  :part_resources,
+                  :part_modules,
+                  :stdout
+
+      def self.call(stepmod_dir, stdout = $stdout)
+        new(stepmod_dir, stdout).call
+      end
+
+      def initialize(stepmod_dir, stdout)
+        @stdout = stdout
+        @stepmod_dir = stepmod_dir
+        @stepmod_path = Pathname.new(stepmod_dir).realpath
+        @general_concepts = Glossarist::Collection.new
+        @resource_concepts = Glossarist::Collection.new
+        @parsed_bibliography = []
+        @part_concepts = []
+        @part_resources = []
+        @part_modules = []
+        @encountered_terms = {}
+      end
+
+      def log(message)
+        stdout.puts "[stepmod-utils] #{message}"
+      end
+
+      def term_special_category(bibdata)
+        case bibdata.part.to_i
+        when 41, 42, 43, 44, 45, 46, 47, 51
+          true
+        when [56..112]
+          true
+        else
+          false
+        end
+      end
+
+      def call
+        # If we are using the stepmod CVS repository, provide the revision number per file
+        @cvs_mode = if Dir.exists?(stepmod_path.join("CVS"))
+                      require "ptools"
+                      # ptools provides File.which
+                      File.which("cvs")
+                    end
+
+        log "INFO: STEPmod directory set to #{stepmod_dir}."
+
+        if cvs_mode
+          log "INFO: STEPmod directory is a \
+            CVS repository and will detect revisions."
+          log "INFO: [CVS] Detecting file revisions can be slow, \
+            please be patient!"
+        else
+          log "INFO: STEPmod directory is not a CVS repository, \
+            skipping revision detection."
+        end
+
+        log "INFO: Detecting paths..."
+
+        repo_index = Nokogiri::XML(File.read(stepmod_path.join("repository_index.xml"))).root
+
+        files = []
+
+        # add module paths
+        repo_index.xpath("//module").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
+          path = Pathname.new("#{stepmod_dir}/modules/#{x['name']}/module.xml")
+          files << path if File.exists? path
+        end
+
+        # add resource_docs paths
+        repo_index.xpath("//resource_doc").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
+          path = Pathname.new("#{stepmod_dir}/resource_docs/#{x['name']}/resource.xml")
+          files << path if File.exists? path
+        end
+
+        # add business_object_models paths
+        repo_index.xpath("//business_object_model").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
+          path = Pathname.new("#{stepmod_dir}/business_object_models/#{x['name']}/business_object_model.xml")
+          files << path if File.exists? path
+        end
+
+        # add application_protocols paths
+        repo_index.xpath("//application_protocol").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
+          path = Pathname.new("#{stepmod_dir}/application_protocols/#{x['name']}/application_protocol.xml")
+          files << path if File.exists? path
+        end
+
+        files.sort!.uniq!
+        process_term_files(files)
+
+        [
+          general_concepts,
+          resource_concepts,
+          parsed_bibliography,
+          part_concepts,
+          part_resources,
+          part_modules,
+        ]
+      end
+
+      private
+
+      def process_term_files(files)
+        parsed_schema_names = {}
+        files.each do |file_path|
+          file_path = file_path.realpath
+          fpath = file_path.relative_path_from(stepmod_path)
+
+          log "INFO: Processing XML file #{fpath}"
+          current_document = Nokogiri::XML(File.read(file_path)).root
+
+          bibdata = nil
+          begin
+            bibdata = Stepmod::Utils::Bibdata.new(document: current_document)
+          rescue StandardError
+            log "WARNING: Unknown file #{fpath}, skipped"
+            next
+          end
+
+          unless ACCEPTED_STAGES.include? bibdata.doctype
+            log "INFO: skipped #{bibdata.docid} as it is not \
+              one of (#{ACCEPTED_STAGES.join(', ')})."
+            next
+          end
+
+          if bibdata.part.to_s.empty?
+            log "FATAL: missing `part` attribute: #{fpath}"
+            log "INFO: skipped #{bibdata.docid} as it is missing `part` attribute."
+            next
+          end
+
+          revision_string = "\n// CVS: revision not detected"
+          if cvs_mode
+            # Run `cvs status` to find out version
+
+            log "INFO: Detecting CVS revision..."
+            Dir.chdir(stepmod_path) do
+              status = `cvs status #{fpath}`
+
+              unless status.empty?
+                working_rev = status.split(/\n/).grep(/Working revision:/)
+                  .first.match(/revision:\s+(.+)$/)[1]
+                repo_rev = status.split(/\n/).grep(/Repository revision:/)
+                  .first.match(/revision:\t(.+)\t/)[1]
+                log "INFO: CVS working rev (#{working_rev}), \
+                  repo rev (#{repo_rev})"
+                revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
+                  "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
+              end
+            end
+          end
+
+          # read definitions
+          current_part_concepts = Glossarist::Collection.new
+          definition_index = 0
+          current_document.xpath("//definition").each do |definition|
+            definition_index += 1
+            term_id = definition["id"]
+            unless term_id.nil?
+              if encountered_terms[term_id]
+                log "FATAL: Duplicated term with id: #{term_id}, #{fpath}"
+              end
+              encountered_terms[term_id] = true
+            end
+
+            # Assume that definition is located in clause 3 of the ISO document
+            # in order. We really don't have a good reference here.
+            ref_clause = "3.#{definition_index}"
+
+            concept = Stepmod::Utils::Concept.parse(
+              definition,
+              reference_anchor: bibdata.anchor,
+              reference_clause: ref_clause,
+              file_path: fpath + revision_string,
+            )
+            next unless concept
+
+            if term_special_category(bibdata)
+              # log "INFO: this part is special"
+              find_or_initialize_concept(current_part_concepts, concept)
+            else
+              # log "INFO: this part is generic"
+              find_or_initialize_concept(general_concepts, concept)
+            end
+
+            parsed_bibliography << bibdata
+          end
+
+          current_part_resources = Glossarist::Collection.new
+          current_part_modules_arm = {}
+          current_part_modules_mim = {}
+
+          log "INFO: FILE PATH IS #{file_path}"
+          case file_path.to_s
+          when /resource.xml$/
+            log "INFO: Processing resource.xml for #{file_path}"
+            # Assumption: every schema is only linked by a single resource_docs document.
+            current_document.xpath("//schema").each do |schema_node|
+              schema_name = schema_node["name"]
+              if parsed_schema_names[schema_name]
+                log "ERROR: We have encountered this schema before: \
+                  #{schema_name} from path \
+                  #{parsed_schema_names[schema_name]}, now at #{file_path}"
+                next
+              else
+                parsed_schema_names[schema_name] = file_path
+              end
+
+              Dir["#{stepmod_path}/resources/#{schema_name}/descriptions.xml"].each do |description_xml_path|
+                log "INFO: Processing resources schema #{description_xml_path}"
+                description_document = Nokogiri::XML(File.read(description_xml_path)).root
+                description_document.xpath("//ext_description").each do |ext_description|
+                  # log "INFO: Processing linkend[#{ext_description['linkend']}]"
+
+                  concept = Stepmod::Utils::Concept.parse(
+                    ext_description,
+                    reference_anchor: bibdata.anchor,
+                    reference_clause: nil,
+                    file_path: Pathname.new(description_xml_path)
+                                .relative_path_from(stepmod_path),
+                  )
+                  next unless concept
+
+                  if term_special_category(bibdata)
+                    # log "INFO: this part is special"
+                    find_or_initialize_concept(current_part_resources, concept)
+                  else
+                    # log "INFO: this part is generic"
+                    find_or_initialize_concept(resource_concepts, concept)
+                  end
+
+                  parsed_bibliography << bibdata
+                end
+              end
+            end
+
+          when /module.xml$/
+            log "INFO: Processing module.xml for #{file_path}"
+            # Assumption: every schema is only linked by a single module document.
+            # puts current_document.xpath('//module').length
+            schema_name = current_document.xpath("//module").first["name"]
+            if parsed_schema_names[schema_name]
+              log "ERROR: We have encountered this schema before: \
+                #{schema_name} from path #{parsed_schema_names[schema_name]}, \
+                  now at #{file_path}"
+              next
+            else
+              parsed_schema_names[schema_name] = file_path
+            end
+
+            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/arm_descriptions.xml"
+            log "INFO: Processing modules schema #{description_xml_path}"
+
+            if File.exists?(description_xml_path)
+              description_document = Nokogiri::XML(
+                File.read(description_xml_path),
+              )
+                .root
+              description_document.xpath("//ext_description").each do |ext_description|
+                linkend_schema = ext_description["linkend"].split(".").first
+                concept = Stepmod::Utils::Concept.parse(
+                  ext_description,
+                  reference_anchor: bibdata.anchor,
+                  reference_clause: nil,
+                  file_path: Pathname.new(description_xml_path)
+                              .relative_path_from(stepmod_path),
+                )
+                next unless concept
+
+                current_part_modules_arm[linkend_schema] ||= Glossarist::Collection.new
+                find_or_initialize_concept(
+                  current_part_modules_arm[linkend_schema], concept
+                )
+                # puts part_modules_arm.inspect
+                parsed_bibliography << bibdata
+              end
+            end
+
+            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/mim_descriptions.xml"
+            log "INFO: Processing modules schema #{description_xml_path}"
+
+            if File.exists?(description_xml_path)
+              description_document = Nokogiri::XML(
+                File.read(description_xml_path),
+              )
+                .root
+              description_document.xpath("//ext_description").each do |ext_description|
+                linkend_schema = ext_description["linkend"].split(".").first
+
+                concept = Stepmod::Utils::Concept.parse(
+                  ext_description,
+                  reference_anchor: bibdata.anchor,
+                  reference_clause: nil,
+                  file_path: Pathname
+                              .new(description_xml_path)
+                              .relative_path_from(stepmod_path),
+                )
+                next unless concept
+
+                current_part_modules_mim[linkend_schema] ||=
+                  Glossarist::Collection.new
+                find_or_initialize_concept(
+                  current_part_modules_mim[linkend_schema], concept
+                )
+
+                parsed_bibliography << bibdata
+              end
+            end
+
+          end
+
+          log "INFO: Completed processing XML file #{fpath}"
+          if current_part_concepts.to_a.empty?
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
+              because it contains no concepts."
+          elsif current_part_concepts.to_a.length < 3
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
+              because it only has #{current_part_concepts.to_a.length} terms."
+
+            current_part_concepts.to_a.each do |x|
+              general_concepts.store(x)
+            end
+          else
+            unless current_part_concepts.to_a.empty?
+              part_concepts << [bibdata,
+                                current_part_concepts]
+            end
+          end
+          unless current_part_resources.to_a.empty?
+            part_resources << [bibdata,
+                               current_part_resources]
+          end
+          if (current_part_modules_arm.to_a.size +
+              current_part_modules_mim.to_a.size).positive?
+            part_modules << [bibdata, current_part_modules_arm,
+                             current_part_modules_mim]
+          end
+        end
+      end
+
+      def find_or_initialize_concept(collection, localized_concept)
+        concept = collection
+          .store(Glossarist::Concept.new(id: SecureRandom.uuid))
+        concept.add_l10n(localized_concept)
+      end
+    end
+  end
+end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.2"
+    VERSION = "0.3.8".freeze
   end
 end

data/stepmod-utils.gemspec

@@ -1,4 +1,4 @@
-require_relative 'lib/stepmod/utils/version'
+require_relative "lib/stepmod/utils/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "stepmod-utils"
@@ -22,16 +22,22 @@ Gem::Specification.new do |spec|
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "thor", ">= 0.20.3"
-  spec.add_runtime_dependency "reverse_adoc", ">= 0.2.9"
-  spec.add_runtime_dependency "ptools", '~> 1.3'
   spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "glossarist", "~> 0.1.0"
+  spec.add_runtime_dependency "ptools"
+  spec.add_runtime_dependency "reverse_adoc", ">= 0.2.9"
+  spec.add_runtime_dependency "thor", ">= 0.20.3"
   spec.add_development_dependency "byebug", "~> 11.1"
+  spec.add_development_dependency "rubocop", "1.12"
+  spec.add_development_dependency "rubocop-performance"
+  spec.add_development_dependency "rubocop-rails"
 end