stepmod-utils 0.3.17 → 0.3.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8be21fe4d04046487ecb36fdebeb74f8dbbb7bda473cbb1b3451b62b15d62a0
-  data.tar.gz: f3005c150d30d5c8aac9b77f9221cfcfc575e0d541208152087f6ff88623375d
+  metadata.gz: 2a0c01510a286c5aa7e3b0a32a910b7b3637726e01cb7433f8552ef6758f3fa5
+  data.tar.gz: 0a3fd286cb1f42a4d5cb52989616cb36cc39cffd49c0cd6a7219685b9b83dfba
 SHA512:
-  metadata.gz: b004697a0c9eeecc570ba7e184fd7165fa991be9b0046f13305106a6cbdff123aca57299c340ce72fb95214ad878a0d0aff140af5e07308070d1c8a336c09921
-  data.tar.gz: 9aaa80fff000066c954b660592626a9ee0132cfca173c0d93aeb572211adf1929d79dc1f1336858725b031b287d1a9d07f9e4d79e45b8d4a4727819fbe303b9f
+  metadata.gz: 4cca0b6de2ee2109e0f11f144406a14f58a54cbd2030ae15180d897a923bf8c98de36b9f8c1d59445a2cde22a1cc2c9ac70474f6b1262fb6a7faa4bec460a631
+  data.tar.gz: c4c78c48c121619e7e47fda0a86caeaec622535cd5c57468e1316998fa4ab7a2c73549fe227c0a7df2bde4ea69f32852b52018304cfabe6c129241f8a2b54484

data/exe/stepmod-extract-terms

@@ -153,10 +153,13 @@ end
 
 log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
 
-part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
+part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
+  bibdata.part.to_i
+end.each do |(bibdata, part_modules_arm, part_modules_mim)|
   fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
   File.open(fn, "w") do |file|
     file.puts("")
+
     unless part_modules_arm.empty?
       schema_name = part_modules_arm.first.first
       concepts = part_modules_arm.first.last.to_a.map do |n|
@@ -181,6 +184,7 @@ part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
       file.puts(replace_images(concepts.map(&:to_mn_adoc).join("\n")))
     end
   end
+
   log "INFO: written to: #{fn}"
 end
 

data/lib/stepmod/utils/concept.rb

@@ -10,6 +10,8 @@ module Stepmod
         file_path
       )
 
+      # TODO: converted_definition is not supposed to be an attribute, it is
+      # supposed to be a method!
       class << self
         def parse(definition_xml, reference_anchor:, reference_clause:, file_path:, language_code: "en")
           converted_definition = Stepmod::Utils::StepmodDefinitionConverter.convert(
@@ -35,6 +37,8 @@ module Stepmod
             ).strip
           end
 
+          # TODO: `designations:` should include the `alt:[...]` terms here,
+          # they are now only included in definition_xml_converted_definition.
           new(
             designations: [designation],
             definition: definition,

data/lib/stepmod/utils/terms_extractor.rb

@@ -26,7 +26,8 @@ module Stepmod
                   :part_concepts,
                   :part_resources,
                   :part_modules,
-                  :stdout
+                  :stdout,
+                  :git_rev
 
       def self.call(stepmod_dir, index_path, stdout = $stdout)
         new(stepmod_dir, index_path, stdout).call
@@ -65,6 +66,12 @@ module Stepmod
         log "INFO: STEPmod directory set to #{stepmod_dir}."
         log "INFO: Detecting paths..."
 
+        # Run `cvs status` to find out version
+        log "INFO: Detecting Git SHA..."
+        Dir.chdir(stepmod_path) do
+          @git_rev = `git rev-parse HEAD` || nil
+        end
+
         repo_index = Nokogiri::XML(File.read(@index_path)).root
 
         files = []
@@ -134,8 +141,8 @@ module Stepmod
           end
 
           unless ACCEPTED_STAGES.include? bibdata.doctype
-            log "INFO: skipped #{bibdata.docid} as it is not \
-              one of (#{ACCEPTED_STAGES.join(', ')})."
+            log "INFO: skipped #{bibdata.docid} as it is not " \
+              "one of (#{ACCEPTED_STAGES.join(', ')})."
             next
           end
 
@@ -145,23 +152,9 @@ module Stepmod
             next
           end
 
-          revision_string = ""
-
-          # Run `cvs status` to find out version
-          log "INFO: Detecting Git SHA..."
-          Dir.chdir(stepmod_path) do
-            git_sha = `git rev-parse HEAD`
-
-            unless git_sha.empty?
-              revision_string = "\n// Git: SHA #{git_sha}"
-            end
-          end
-
           # read definitions
           current_part_concepts = Glossarist::Collection.new
-          definition_index = 0
-          current_document.xpath("//definition").each do |definition|
-            definition_index += 1
+          current_document.xpath("//definition").each_with_index do |definition, definition_index|
             term_id = definition["id"]
             unless term_id.nil?
               if encountered_terms[term_id]
@@ -178,7 +171,7 @@ module Stepmod
               definition,
               reference_anchor: bibdata.anchor,
               reference_clause: ref_clause,
-              file_path: fpath + revision_string,
+              file_path: fpath,
             )
             next unless concept
 
@@ -197,10 +190,10 @@ module Stepmod
           current_part_modules_arm = {}
           current_part_modules_mim = {}
 
-          log "INFO: FILE PATH IS #{file_path}"
+          # log "INFO: FILE PATH IS #{file_path}"
           case file_path.to_s
           when /resource.xml$/
-            log "INFO: Processing resource.xml for #{file_path}"
+            log "INFO: Processing resource.xml for #{fpath}"
 
             current_document.xpath("//schema").each do |schema_node|
               schema_name = schema_node["name"]
@@ -238,7 +231,7 @@ module Stepmod
                     id: "#{reference_anchor}.#{reference_clause}",
                     reference_anchor: reference_anchor,
                     reference_clause: reference_clause,
-                    file_path: Pathname.new(file_path)
+                    file_path: Pathname.new(exp_annotated_path)
                                 .relative_path_from(stepmod_path),
                     language_code: "en",
                   )
@@ -259,7 +252,7 @@ module Stepmod
             end
 
           when /module.xml$/
-            log "INFO: Processing module.xml for #{file_path}"
+            log "INFO: Processing module.xml for #{fpath}"
             # Assumption: every schema is only linked by a single module document.
             # puts current_document.xpath('//module').length
             schema_name = current_document.xpath("//module").first["name"]
@@ -272,101 +265,36 @@ module Stepmod
               parsed_schema_names[schema_name] = file_path
             end
 
-            exp_annotated_path =
-              "#{stepmod_path}/modules/#{schema_name}/arm_annotated.exp"
-
-            log "INFO: Processing modules schema #{exp_annotated_path}"
-
-            if File.exists?(exp_annotated_path)
-              repo = Expressir::Express::Parser.from_file(exp_annotated_path)
-
-              repo.schemas.each do |schema|
-                schema.entities.each do |entity|
-                  old_definition = entity.remarks.first
-
-                  domain = "application module: #{schema.id}"
-                  entity_definition = generate_entity_definition(entity, domain, old_definition)
-
-                  reference_anchor = bibdata.anchor
-                  reference_clause = nil
-
-                  concept = Stepmod::Utils::Concept.new(
-                    designations: [entity.id],
-                    definition: old_definition,
-                    converted_definition: entity_definition,
-                    id: "#{reference_anchor}.#{reference_clause}",
-                    reference_anchor: reference_anchor,
-                    reference_clause: reference_clause,
-                    file_path: Pathname.new(file_path)
-                                .relative_path_from(stepmod_path),
-                    language_code: "en",
-                  )
-
-                  next unless concept
+            arm_schema, arm_concepts = parse_annotated_module(
+              type: :arm,
+              stepmod_path: stepmod_path,
+              path: "modules/#{schema_name}/arm_annotated.exp",
+              bibdata: bibdata
+            )
 
-                  current_part_modules_arm[schema.id] ||=
-                    Glossarist::Collection.new
-                  find_or_initialize_concept(
-                    current_part_modules_arm[schema.id], concept
-                  )
+            mim_schema, mim_concepts = parse_annotated_module(
+              type: :mim,
+              stepmod_path: stepmod_path,
+              path: "modules/#{schema_name}/mim_annotated.exp",
+              bibdata: bibdata
+            )
 
-                  # puts part_modules_arm.inspect
-                  parsed_bibliography << bibdata
-                end
-              end
+            if arm_concepts.to_a.size > 0
+              current_part_modules_arm[arm_schema] = arm_concepts
             end
 
-            mim_exp_annotated_path = "#{stepmod_path}/modules/#{schema_name}/mim_annotated.exp"
-
-            log "INFO: Processing modules schema #{mim_exp_annotated_path}"
-
-            if File.exists?(mim_exp_annotated_path)
-              repo = Expressir::Express::Parser.from_file(mim_exp_annotated_path)
-
-              repo.schemas.each do |schema|
-                schema.entities.each do |entity|
-                  old_definition = entity.remarks.first
-
-                  domain = "application module: #{schema.id}"
-                  definition = generate_entity_definition(entity, domain, old_definition)
-
-                  reference_anchor = bibdata.anchor
-                  reference_clause = nil
-
-                  concept = Stepmod::Utils::Concept.new(
-                    designations: [entity.id],
-                    definition: old_definition,
-                    converted_definition: definition,
-                    id: "#{reference_anchor}.#{reference_clause}",
-                    reference_anchor: reference_anchor,
-                    reference_clause: reference_clause,
-                    file_path: Pathname.new(file_path)
-                                .relative_path_from(stepmod_path),
-                    language_code: "en",
-                  )
-
-                  next unless concept
-
-                  current_part_modules_mim[schema.id] ||=
-                    Glossarist::Collection.new
-                  find_or_initialize_concept(
-                    current_part_modules_mim[schema.id], concept
-                  )
-
-                  parsed_bibliography << bibdata
-                end
-              end
+            if mim_concepts.to_a.size > 0
+              current_part_modules_mim[mim_schema] = mim_concepts
             end
-
           end
 
           log "INFO: Completed processing XML file #{fpath}"
           if current_part_concepts.to_a.empty?
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
-              because it contains no concepts."
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
+              "because it contains no concepts."
           elsif current_part_concepts.to_a.length < 3
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
-              because it only has #{current_part_concepts.to_a.length} terms."
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
+              "because it only has #{current_part_concepts.to_a.length} terms."
 
             current_part_concepts.to_a.each do |x|
               general_concepts.store(x)
@@ -377,38 +305,201 @@ module Stepmod
                                 current_part_concepts]
             end
           end
+
           unless current_part_resources.to_a.empty?
             part_resources << [bibdata,
                                current_part_resources]
           end
+
           if (current_part_modules_arm.to_a.size +
               current_part_modules_mim.to_a.size).positive?
+
             part_modules << [bibdata, current_part_modules_arm,
                              current_part_modules_mim]
+            parsed_bibliography << bibdata
           end
+
         end
       end
 
-      def find_or_initialize_concept(collection, localized_concept)
-        concept_id = SecureRandom.uuid
+      def parse_annotated_module(type:, stepmod_path:, path:, bibdata:)
+        log "INFO: parse_annotated_module: Processing modules schema #{path}"
+
+        fpath = File.join(stepmod_path, path)
+
+        unless File.exists?(fpath)
+          log "ERROR: parse_annotated_module: No module schema exists at #{fpath}."
+          return
+        end
+
+        repo = Expressir::Express::Parser.from_file(fpath)
+
+        unless repo
+          log "ERROR: parse_annotated_module: failed to parse EXPRESS file at #{path}."
+          return
+        end
+
+        # See: metanorma/iso-10303-2#90
+        domain_prefix = case type
+        when :mim
+          "application module"
+        when :arm
+          "application object"
+        end
 
+        if repo.schemas.length > 1
+          raise StandardError.new(
+            "ERROR: FATAL: #{fpath} contains more than one schema:" +
+            "#{repo.schemas.map(&:id).join(", ")} (not supposed to happen!!)"
+          )
+        end
+
+        schema = repo.schemas.first
+        collection = Glossarist::Collection.new
+        domain = "#{domain_prefix}: #{schema.id}"
+
+        schema.entities.each do |entity|
+          old_definition = entity.remarks.first
+          new_definition = generate_entity_definition(entity, domain, old_definition)
+
+          concept = Stepmod::Utils::Concept.new(
+            designations: [entity.id],
+            definition: old_definition,
+            converted_definition: new_definition,
+            # TODO: Find a proper ID for this
+            id: "#{bibdata.anchor}.",
+            reference_anchor: bibdata.anchor,
+            reference_clause: nil,
+            file_path: path,
+            language_code: "en",
+          )
+
+          next unless concept
+          find_or_initialize_concept(collection, concept)
+        end
+
+        [schema.id, collection]
+      end
+
+      def find_or_initialize_concept(collection, localized_concept)
         concept = collection
-          .store(Glossarist::Concept.new(id: concept_id))
+          .store(Glossarist::Concept.new(id: SecureRandom.uuid))
         concept.add_l10n(localized_concept)
       end
 
+      def combine_paragraphs(full_paragraph, next_paragraph)
+        # If full_paragraph already contains a period, extract that.
+        if m = full_paragraph.match(/\A(?<inner_first>[^\n]*?\.)\s/)
+          # puts "CONDITION 1"
+          if m[:inner_first]
+            return m[:inner_first]
+          else
+            return full_paragraph
+          end
+        end
+
+        # If full_paragraph ends with a period, this is the last.
+        if full_paragraph =~ /\.\s*\Z/
+          # puts "CONDITION 2"
+          return full_paragraph
+        end
+
+        # If next_paragraph is a list
+        if next_paragraph.match(/\A\*/)
+          # puts "CONDITION 3"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # If next_paragraph is a continuation of a list
+        if next_paragraph.match(/\Awhich/) || next_paragraph.match(/\Athat/)
+          # puts "CONDITION 4"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # puts "CONDITION 5"
+        full_paragraph
+      end
+
+      def trim_definition(definition)
+        # Unless the first paragraph ends with "between" and is followed by a
+        # list, don't split
+        paragraphs = definition.split("\n\n")
+
+        # puts paragraphs.inspect
+
+        first_paragraph = paragraphs.first
+
+        if paragraphs.length > 1
+          combined = paragraphs[1..-1].inject(first_paragraph) do |acc, p|
+            combine_paragraphs(acc, p)
+          end
+        else
+          combined = combine_paragraphs(first_paragraph, "")
+        end
+
+        # puts "combined--------- #{combined}"
+
+        # Remove comments until end of line
+        combined = combined + "\n"
+        combined.gsub!(/\n\/\/.*?\n/, "\n")
+        combined.strip!
+
+        combined
+        # # TODO: If the definition contains a list immediately after the first paragraph, don't split
+        # return definition if definition =~ /\n\* /
+
+        # unless (
+        #   first_paragraph =~ /between:?\s*\Z/ ||
+        #   first_paragraph =~ /include:?\s*\Z/ ||
+        #   first_paragraph =~ /of:?\s*\Z/ ||
+        #   first_paragraph =~ /[:;]\s*\Z/
+        #   ) &&
+        #   definition =~ /\n\n\*/
+
+        #   # Only taking the first paragraph of the definition
+        #   first_paragraph
+        # end
+      end
+
+      def entity_name_to_text(entity_id)
+        entity_id.downcase.gsub(/_/, " ")
+      end
+
       # rubocop:disable Layout/LineLength
       def generate_entity_definition(entity, domain, old_definition)
         return "" if entity.nil?
 
-        entity_text = if entity.subtype_of.size.zero?
-                        "entity data type that represents " +
-                        entity.id.indefinite_article + " **#{entity.id}** entity"
-                      else
-                        "entity data type that is a type of "+
-                        "**#{entity.subtype_of.map(&:id).join('** and **')}** that represents " +
-                        entity.id.indefinite_article + " **#{entity.id}** entity"
-                      end
+        # See: metanorma/iso-10303-2#90
+        # TODO: This is not DRY in case we have to further customize
+        entity_text = if domain_type = domain.match(/\A(application.*?):/)
+
+          if entity.subtype_of.size.zero?
+            "#{domain_type[1]} that represents the " +
+            "{{#{entity.id},#{entity_name_to_text(entity.id)}}} entity"
+          else
+            entity_subtypes = entity.subtype_of.map do |e|
+              "{{#{e.id},#{entity_name_to_text(e.id)}}}"
+            end
+            "#{domain_type[1]} that is a type of " +
+            "#{entity_subtypes.join(' and ')} that represents the " +
+            "{{#{entity.id},#{entity_name_to_text(entity.id)}}} entity"
+          end
+
+        else
+
+          # Not "application object" or "application module"
+          if entity.subtype_of.size.zero?
+            "entity data type that represents " +
+            entity.id.indefinite_article + " {{#{entity.id}}} entity"
+          else
+            entity_subtypes = entity.subtype_of.map do |e|
+              "{{#{e.id}}}"
+            end
+            "entity data type that is a type of "+
+            "#{entity_subtypes.join(' and ')} that represents " +
+            entity.id.indefinite_article + " {{#{entity.id}}} entity"
+          end
+        end
 
         definition = <<~DEFINITION
           === #{entity.id}
@@ -418,11 +509,14 @@ module Stepmod
 
         DEFINITION
 
+        # If there is a definition, we add it as the first NOTE
         unless old_definition.nil? || old_definition.blank?
+          old_definition = trim_definition(old_definition)
+
           definition << <<~OLD_DEFINITION
             [NOTE]
             --
-            #{old_definition&.strip}
+            #{old_definition.strip}
             --
           OLD_DEFINITION
         end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.17".freeze
+    VERSION = "0.3.20".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.17
+  version: 0.3.20
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-07 00:00:00.000000000 Z
+date: 2022-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -253,7 +253,7 @@ homepage: https://github.com/metanorma/stepmod-utils
 licenses:
 - BSD-2-Clause
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -268,8 +268,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.1.6
+signing_key: 
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.
 test_files: []