stepmod-utils 0.3.16 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e40fc2ba271e4bd8e0859cb307831ead927b7bc12154b5845f528694ad18c8f8
-  data.tar.gz: e1b6f691a70d1e6531a7f841b5a998d49d6c376a84a9da340246b2920bd8459d
+  metadata.gz: 4541a1fb0b4b3671d2aadf8cb833e93ca8b45f0c591b0da2df55e4614c2bcbf4
+  data.tar.gz: dbf98829e92bfcd54b468691011ab0de3781cf515b4d4b511ec394846f556ac6
 SHA512:
-  metadata.gz: e5e62c15d95982deb33e03af65b9720cdb34746adfa36bf5945a2823fb0be9800beff3e8cf4b3e4acd34ebf940bacb0ff2b4a83b31af27e8d01cba2599fcc537
-  data.tar.gz: c9efdfa5eed962291c9454227e8d2c7c22b4ba665c88745d059f77cdfb5ae2bdb414d32d7c1ee957aec10db634c1f0fbc388851360ae4edb9bf9d721f740dca3
+  metadata.gz: 8410dd3702701b592aa67af784813058dcf2299ffe9dd60d45cab59b7035952111e17144b3b3b446e03135ae84a2a7ac5467f155b92ff768aa155d96437f0387
+  data.tar.gz: ea2fb98127cb384a70bd5aaf5febab17a92460170c820b250c52c0dca884bdc8d298f5ecdcf54f52ded4d1d22e75cbbce0fc65582499a04413e5e95cc8eb6b85

data/.github/workflows/rake.yml

@@ -10,6 +10,6 @@ on:
 
 jobs:
   rake:
-    uses: metanorma/metanorma-build-scripts/.github/workflows/generic-rake.yml@main
+    uses: metanorma/ci/.github/workflows/generic-rake.yml@main
     secrets:
       pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/.github/workflows/release.yml

@@ -15,10 +15,10 @@ on:
 
 jobs:
   release:
-    uses: metanorma/metanorma-build-scripts/.github/workflows/rubygems-release.yml@main
+    uses: metanorma/ci/.github/workflows/rubygems-release.yml@main
     with:
       next_version: ${{ github.event.inputs.next_version }}
-      event_name: ${{ github.event_name }}
     secrets:
       rubygems-api-key: ${{ secrets.METANORMA_CI_RUBYGEMS_API_KEY }}
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
 

data/exe/stepmod-annotate-all

@@ -27,15 +27,20 @@ files.each_slice(MAX_QUEUE_SIZE) do |batch|
   batch.each do |file|
     pool.post do
       puts("#{Thread.current.object_id}: Queued processing #{file}")
-      annotated = Stepmod::Utils::StepmodFileAnnotator.new(express_file: file,
-                                                           resource_docs_cache_file: resource_docs_cache_file, stepmod_dir: stepmod_dir).call
-      File.open(
-        File.join(File.dirname(file),
-                  "#{File.basename(file, '.exp')}_annotated.exp"), "w"
-      ) do |file|
+      annotated = Stepmod::Utils::StepmodFileAnnotator.new(
+        express_file: file,
+        resource_docs_cache_file: resource_docs_cache_file,
+        stepmod_dir: stepmod_dir
+      ).call
+
+      annotated_file_name = "#{File.basename(file, '.exp')}_annotated.exp"
+      annotated_file_path = File.join(File.dirname(file), annotated_file_name)
+
+      File.open(annotated_file_path, "w") do |file|
         file.puts(annotated)
       end
-      puts("#{Thread.current.object_id}: Done processing #{file}")
+
+      puts("#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}.")
     end
   end
   pool.shutdown

data/exe/stepmod-extract-terms

@@ -106,28 +106,29 @@ def replace_images(content)
   content
 end
 
-part_concepts.each do |(bibdata, current_part_concepts)|
-  current_part_concepts = current_part_concepts.to_a.map do |n|
-    n.localizations["en"]
-  end
-  fn = "03x-stepmod-#{bibdata.part}.adoc"
-  File.open(fn, "w") do |file|
-    file.puts("== #{part_to_title(bibdata)}\n\n")
-    file.puts(replace_images(current_part_concepts.map(&:to_mn_adoc).join("\n")))
-  end
-  log "INFO: written to: #{fn}"
-end
-
-File.open("03x-stepmod.adoc", "w") do |file|
-  part_concepts.sort_by do |(bibdata, current_part_concepts)|
-    bibdata.part.to_i
-  end.each do |(bibdata, current_part_concepts)|
-    fn = "03x-stepmod-#{bibdata.part}.adoc"
-    file.puts("\ninclude::#{fn}[]\n")
-  end
-end
-
-log "INFO: written summary file to: 03x-stepmod.adoc"
+## Skip all Clause 3 terms
+# part_concepts.each do |(bibdata, current_part_concepts)|
+#   current_part_concepts = current_part_concepts.to_a.map do |n|
+#     n.localizations["en"]
+#   end
+#   fn = "03x-stepmod-#{bibdata.part}.adoc"
+#   File.open(fn, "w") do |file|
+#     file.puts("== #{part_to_title(bibdata)}\n\n")
+#     file.puts(replace_images(current_part_concepts.map(&:to_mn_adoc).join("\n")))
+#   end
+#   log "INFO: written to: #{fn}"
+# end
+
+# File.open("03x-stepmod.adoc", "w") do |file|
+#   part_concepts.sort_by do |(bibdata, current_part_concepts)|
+#     bibdata.part.to_i
+#   end.each do |(bibdata, current_part_concepts)|
+#     fn = "03x-stepmod-#{bibdata.part}.adoc"
+#     file.puts("\ninclude::#{fn}[]\n")
+#   end
+# end
+
+# log "INFO: written summary file to: 03x-stepmod.adoc"
 
 part_resources.each do |(bibdata, current_part_resources)|
   current_part_resources = current_part_resources.to_a.map do |n|
@@ -152,10 +153,13 @@ end
 
 log "INFO: written summary file to: 04x-stepmod-entities-resources.adoc"
 
-part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
+part_modules.sort_by do |(bibdata, part_modules_arm, part_modules_mim)|
+  bibdata.part.to_i
+end.each do |(bibdata, part_modules_arm, part_modules_mim)|
   fn = "05x-stepmod-entities-modules-#{bibdata.part}.adoc"
   File.open(fn, "w") do |file|
     file.puts("")
+
     unless part_modules_arm.empty?
       schema_name = part_modules_arm.first.first
       concepts = part_modules_arm.first.last.to_a.map do |n|
@@ -180,6 +184,7 @@ part_modules.each do |(bibdata, part_modules_arm, part_modules_mim)|
       file.puts(replace_images(concepts.map(&:to_mn_adoc).join("\n")))
     end
   end
+
   log "INFO: written to: #{fn}"
 end
 
@@ -194,17 +199,18 @@ end
 
 log "INFO: written summary file to: 05x-stepmod-entities-modules.adoc"
 
-File.open("031-stepmod-general.adoc", "w") do |file|
-  file.puts(
-    replace_images(
-      general_concepts.to_a.map do |n|
-        n.localizations["en"]
-      end.map(&:to_mn_adoc).join("\n")
-    )
-  )
-end
-
-log "INFO: written to: 031-stepmod-general.adoc"
+## Skipping all Clause 3 terms
+# File.open("031-stepmod-general.adoc", "w") do |file|
+#   file.puts(
+#     replace_images(
+#       general_concepts.to_a.map do |n|
+#         n.localizations["en"]
+#       end.map(&:to_mn_adoc).join("\n")
+#     )
+#   )
+# end
+
+# log "INFO: written to: 031-stepmod-general.adoc"
 
 File.open("041-stepmod-entities-resources.adoc", "w") do |file|
   file.puts(

data/lib/stepmod/utils/concept.rb

@@ -23,15 +23,6 @@ module Stepmod
 
           return nil if converted_definition.nil? || converted_definition.strip.empty?
 
-          if definition_xml.name == "ext_description"
-            converted_definition = <<~TEXT
-              #{converted_definition}
-
-              NOTE: This term is incompletely defined in this document.
-              Reference <<#{reference_anchor}>> for the complete definition.
-            TEXT
-          end
-
           # https://github.com/metanorma/stepmod-utils/issues/86
 
           # TODO: This portion DOES NOT HANDLE the <synonym> element. WTF??

data/lib/stepmod/utils/converters/express_example.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class ExpressExample < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          <<~TEMPLATE
+            (*"#{state[:schema_and_entity]}.__example"
+            #{treat_children(node, state).strip}
+            *)
+          TEMPLATE
+        end
+      end
+
+      ReverseAdoc::Converters.register :express_example, ExpressExample.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/express_note.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class ExpressNote < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          <<~TEMPLATE
+            (*"#{state[:schema_and_entity]}.__note"
+            #{treat_children(node, state).strip}
+            *)
+          TEMPLATE
+        end
+      end
+
+      ReverseAdoc::Converters.register :express_note, ExpressNote.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/module_ref_express_description.rb

@@ -0,0 +1,14 @@
+module Stepmod
+  module Utils
+    module Converters
+      class ModuleRefExpressDescription < ReverseAdoc::Converters::Base
+        def convert(node, _state = {})
+          parts = node["linkend"].to_s.split(":").last.split(".")
+          "<<express:#{parts.join('.').strip},#{parts.last.strip}>>"
+        end
+      end
+      ReverseAdoc::Converters.register :module_ref,
+                                       ModuleRefExpressDescription.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/stepmod_ext_description.rb

@@ -43,18 +43,17 @@ module Stepmod
 
           domain =  case linkend.first
                     when /_mim$/, /_arm$/
-                      "ISO 10303 application module"
+                      "application module"
                     # when /_schema$/
                     else
-                      "ISO 10303 resource"
+                      "resource"
                     end
 
           <<~TEMPLATE
             === #{linkend.last}
+            #{domain ? "domain:[#{domain}: #{linkend.first}]" : ''}
 
-            #{domain ? "domain:[#{domain}]" : ''}
-
-            #{child_text}
+            #{child_text.strip}
           TEMPLATE
         end
       end

data/lib/stepmod/utils/smrl_description_converter.rb

@@ -41,7 +41,7 @@ module Stepmod
         root || (return "")
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
           Stepmod::Utils::Cleaner.new.tidy(result)
         end
       end

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -59,7 +59,8 @@ module Stepmod
         root || (return "")
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
+
           Stepmod::Utils::Cleaner.new.tidy(result)
         end
       end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -1,6 +1,8 @@
 require "json"
 require "stepmod/utils/smrl_description_converter"
 require "stepmod/utils/smrl_resource_converter"
+require "stepmod/utils/converters/express_note"
+require "stepmod/utils/converters/express_example"
 
 module Stepmod
   module Utils
@@ -64,7 +66,27 @@ module Stepmod
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<ext_descriptions>#{description}</ext_descriptions>"
-          "\n#{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)}"
+          notes = description.xpath("note")
+          examples = description.xpath("example")
+
+          converted_description = <<~DESCRIPTION
+
+            #{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper, no_notes_examples: true)}
+          DESCRIPTION
+
+          converted_examples = examples.map do |example|
+            Stepmod::Utils::Converters::ExpressExample
+              .new
+              .convert(example, schema_and_entity: description["linkend"])
+          end.join
+
+          converted_notes = notes.map do |note|
+            Stepmod::Utils::Converters::ExpressNote
+              .new
+              .convert(note, schema_and_entity: description["linkend"])
+          end.join
+
+          "#{converted_description}#{converted_examples}#{converted_notes}"
         end
       end
 
@@ -77,7 +99,7 @@ module Stepmod
 
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<resource>#{schema}</resource>"
-          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper)}"
+          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper, no_notes_examples: true)}"
         end
       end
     end

data/lib/stepmod/utils/terms_extractor.rb

@@ -3,6 +3,9 @@ require "stepmod/utils/bibdata"
 require "stepmod/utils/concept"
 require "glossarist"
 require "securerandom"
+require "expressir"
+require "expressir/express/parser"
+require "indefinite_article"
 
 ReverseAdoc.config.unknown_tags = :bypass
 
@@ -131,8 +134,8 @@ module Stepmod
           end
 
           unless ACCEPTED_STAGES.include? bibdata.doctype
-            log "INFO: skipped #{bibdata.docid} as it is not \
-              one of (#{ACCEPTED_STAGES.join(', ')})."
+            log "INFO: skipped #{bibdata.docid} as it is not " \
+              "one of (#{ACCEPTED_STAGES.join(', ')})."
             next
           end
 
@@ -197,8 +200,8 @@ module Stepmod
           log "INFO: FILE PATH IS #{file_path}"
           case file_path.to_s
           when /resource.xml$/
-            log "INFO: Processing resource.xml for #{file_path}"
-            # Assumption: every schema is only linked by a single resource_docs document.
+            log "INFO: Processing resource.xml for #{fpath}"
+
             current_document.xpath("//schema").each do |schema_node|
               schema_name = schema_node["name"]
               if parsed_schema_names[schema_name]
@@ -210,19 +213,36 @@ module Stepmod
                 parsed_schema_names[schema_name] = file_path
               end
 
-              Dir["#{stepmod_path}/resources/#{schema_name}/descriptions.xml"].each do |description_xml_path|
-                log "INFO: Processing resources schema #{description_xml_path}"
-                description_document = Nokogiri::XML(File.read(description_xml_path)).root
-                description_document.xpath("//ext_description").each do |ext_description|
-                  # log "INFO: Processing linkend[#{ext_description['linkend']}]"
-
-                  concept = Stepmod::Utils::Concept.parse(
-                    ext_description,
-                    reference_anchor: bibdata.anchor,
-                    reference_clause: nil,
-                    file_path: Pathname.new(description_xml_path)
+              exp_annotated_path =
+                "#{stepmod_path}/resources/#{schema_name}/#{schema_name}_annotated.exp"
+
+              log "INFO: Processing resources schema #{exp_annotated_path}"
+
+              if File.exists?(exp_annotated_path)
+                repo = Expressir::Express::Parser.from_file(exp_annotated_path)
+                schema = repo.schemas.first
+
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  domain = "resource: #{schema.id}"
+                  entity_definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: entity_definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
                                 .relative_path_from(stepmod_path),
+                    language_code: "en",
                   )
+
                   next unless concept
 
                   if term_special_category(bibdata)
@@ -239,7 +259,7 @@ module Stepmod
             end
 
           when /module.xml$/
-            log "INFO: Processing module.xml for #{file_path}"
+            log "INFO: Processing module.xml for #{fpath}"
             # Assumption: every schema is only linked by a single module document.
             # puts current_document.xpath('//module').length
             schema_name = current_document.xpath("//module").first["name"]
@@ -252,62 +272,90 @@ module Stepmod
               parsed_schema_names[schema_name] = file_path
             end
 
-            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/arm_descriptions.xml"
-            log "INFO: Processing modules schema #{description_xml_path}"
-
-            if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(
-                File.read(description_xml_path),
-              )
-                .root
-              description_document.xpath("//ext_description").each do |ext_description|
-                linkend_schema = ext_description["linkend"].split(".").first
-                concept = Stepmod::Utils::Concept.parse(
-                  ext_description,
-                  reference_anchor: bibdata.anchor,
-                  reference_clause: nil,
-                  file_path: Pathname.new(description_xml_path)
-                              .relative_path_from(stepmod_path),
-                )
-                next unless concept
-
-                current_part_modules_arm[linkend_schema] ||= Glossarist::Collection.new
-                find_or_initialize_concept(
-                  current_part_modules_arm[linkend_schema], concept
-                )
-                # puts part_modules_arm.inspect
-                parsed_bibliography << bibdata
+            exp_annotated_path =
+              "#{stepmod_path}/modules/#{schema_name}/arm_annotated.exp"
+
+            log "INFO: Processing modules schema #{exp_annotated_path}"
+
+            if File.exists?(exp_annotated_path)
+              repo = Expressir::Express::Parser.from_file(exp_annotated_path)
+
+              repo.schemas.each do |schema|
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  # See: metanorma/iso-10303-2#90
+                  domain = "application object: #{schema.id}"
+                  entity_definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: entity_definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
+                                .relative_path_from(stepmod_path),
+                    language_code: "en",
+                  )
+
+                  next unless concept
+
+                  current_part_modules_arm[schema.id] ||=
+                    Glossarist::Collection.new
+                  find_or_initialize_concept(
+                    current_part_modules_arm[schema.id], concept
+                  )
+
+                  # puts part_modules_arm.inspect
+                  parsed_bibliography << bibdata
+                end
               end
             end
 
-            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/mim_descriptions.xml"
-            log "INFO: Processing modules schema #{description_xml_path}"
-
-            if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(
-                File.read(description_xml_path),
-              )
-                .root
-              description_document.xpath("//ext_description").each do |ext_description|
-                linkend_schema = ext_description["linkend"].split(".").first
-
-                concept = Stepmod::Utils::Concept.parse(
-                  ext_description,
-                  reference_anchor: bibdata.anchor,
-                  reference_clause: nil,
-                  file_path: Pathname
-                              .new(description_xml_path)
-                              .relative_path_from(stepmod_path),
-                )
-                next unless concept
-
-                current_part_modules_mim[linkend_schema] ||=
-                  Glossarist::Collection.new
-                find_or_initialize_concept(
-                  current_part_modules_mim[linkend_schema], concept
-                )
-
-                parsed_bibliography << bibdata
+            mim_exp_annotated_path = "#{stepmod_path}/modules/#{schema_name}/mim_annotated.exp"
+
+            log "INFO: Processing modules schema #{mim_exp_annotated_path}"
+
+            if File.exists?(mim_exp_annotated_path)
+              repo = Expressir::Express::Parser.from_file(mim_exp_annotated_path)
+
+              repo.schemas.each do |schema|
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  domain = "application module: #{schema.id}"
+                  definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
+                                .relative_path_from(stepmod_path),
+                    language_code: "en",
+                  )
+
+                  next unless concept
+
+                  current_part_modules_mim[schema.id] ||=
+                    Glossarist::Collection.new
+                  find_or_initialize_concept(
+                    current_part_modules_mim[schema.id], concept
+                  )
+
+                  parsed_bibliography << bibdata
+                end
               end
             end
 
@@ -315,11 +363,11 @@ module Stepmod
 
           log "INFO: Completed processing XML file #{fpath}"
           if current_part_concepts.to_a.empty?
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
-              because it contains no concepts."
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
+              "because it contains no concepts."
           elsif current_part_concepts.to_a.length < 3
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
-              because it only has #{current_part_concepts.to_a.length} terms."
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) " \
+              "because it only has #{current_part_concepts.to_a.length} terms."
 
             current_part_concepts.to_a.each do |x|
               general_concepts.store(x)
@@ -330,15 +378,18 @@ module Stepmod
                                 current_part_concepts]
             end
           end
+
           unless current_part_resources.to_a.empty?
             part_resources << [bibdata,
                                current_part_resources]
           end
+
           if (current_part_modules_arm.to_a.size +
               current_part_modules_mim.to_a.size).positive?
             part_modules << [bibdata, current_part_modules_arm,
                              current_part_modules_mim]
           end
+
         end
       end
 
@@ -347,6 +398,170 @@ module Stepmod
           .store(Glossarist::Concept.new(id: SecureRandom.uuid))
         concept.add_l10n(localized_concept)
       end
+
+      def combine_paragraphs(full_paragraph, next_paragraph)
+        # If full_paragraph already contains a period, extract that.
+        if m = full_paragraph.match(/\A(?<inner_first>[^\n]*?\.)\s/)
+          # puts "CONDITION 1"
+          if m[:inner_first]
+            return m[:inner_first]
+          else
+            return full_paragraph
+          end
+        end
+
+        # If full_paragraph ends with a period, this is the last.
+        if full_paragraph =~ /\.\s*\Z/
+          # puts "CONDITION 2"
+          return full_paragraph
+        end
+
+        # If next_paragraph is a list
+        if next_paragraph.match(/\A\*/)
+          # puts "CONDITION 3"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # If next_paragraph is a continuation of a list
+        if next_paragraph.match(/\Awhich/) || next_paragraph.match(/\Athat/)
+          # puts "CONDITION 4"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # puts "CONDITION 5"
+        full_paragraph
+      end
+
+      def trim_definition(definition)
+        # Unless the first paragraph ends with "between" and is followed by a
+        # list, don't split
+        paragraphs = definition.split("\n\n")
+
+        # puts paragraphs.inspect
+
+        first_paragraph = paragraphs.first
+
+        if paragraphs.length > 1
+          combined = paragraphs[1..-1].inject(first_paragraph) do |acc, p|
+            combine_paragraphs(acc, p)
+          end
+        else
+          combined = combine_paragraphs(first_paragraph, "")
+        end
+
+        # puts "combined--------- #{combined}"
+
+        # Remove comments until end of line
+        combined = combined + "\n"
+        combined.gsub!(/\n\/\/.*?\n/, "\n")
+        combined.strip!
+
+        combined
+        # # TODO: If the definition contains a list immediately after the first paragraph, don't split
+        # return definition if definition =~ /\n\* /
+
+        # unless (
+        #   first_paragraph =~ /between:?\s*\Z/ ||
+        #   first_paragraph =~ /include:?\s*\Z/ ||
+        #   first_paragraph =~ /of:?\s*\Z/ ||
+        #   first_paragraph =~ /[:;]\s*\Z/
+        #   ) &&
+        #   definition =~ /\n\n\*/
+
+        #   # Only taking the first paragraph of the definition
+        #   first_paragraph
+        # end
+      end
+
+      def entity_name_to_text(entity_id)
+        entity_id.downcase.gsub(/_/, " ")
+      end
+
+      # rubocop:disable Layout/LineLength
+      def generate_entity_definition(entity, domain, old_definition)
+        return "" if entity.nil?
+
+        # See: metanorma/iso-10303-2#90
+        # TODO: This is not DRY in case we have to further customize
+        entity_text = if domain =~ /\Aapplication object:/
+
+          if entity.subtype_of.size.zero?
+            "application object that represents the " +
+            "{{#{entity.id},#{entity_name_to_text(entity.id)}}} entity"
+          else
+            entity_subtypes = entity.subtype_of.map do |e|
+              "{{#{e.id},#{entity_name_to_text(e.id)}}}"
+            end
+            "application object that is a type of " +
+            "#{entity_subtypes.join(' and ')} that represents the " +
+            "{{#{entity.id},#{entity_name_to_text(entity.id)}}} entity"
+          end
+
+        else
+
+          # Not "application object"
+          if entity.subtype_of.size.zero?
+            "entity data type that represents " +
+            entity.id.indefinite_article + " {{#{entity.id}}} entity"
+          else
+            entity_subtypes = entity.subtype_of.map do |e|
+              "{{#{e.id}}}"
+            end
+            "entity data type that is a type of "+
+            "#{entity_subtypes.join(' and ')} that represents " +
+            entity.id.indefinite_article + " {{#{entity.id}}} entity"
+          end
+        end
+
+        definition = <<~DEFINITION
+          === #{entity.id}
+          domain:[#{domain}]
+
+          #{entity_text}
+
+        DEFINITION
+
+        unless old_definition.nil? || old_definition.blank?
+          old_definition = trim_definition(old_definition)
+
+          definition << <<~OLD_DEFINITION
+            [NOTE]
+            --
+            #{old_definition.strip}
+            --
+          OLD_DEFINITION
+        end
+
+        # We no longer add Notes and Examples to the extracted terms
+        # definition + format_remark_items(entity.remark_items)
+
+        definition
+      end
+
+      def format_remark_items(remark_items)
+        notes = remark_items.detect { |i| i.id == "__note" }&.remarks
+        examples = remark_items.detect { |i| i.id == "__example" }&.remarks
+
+        formatted_notes = format_remarks(notes, "NOTE", "--")
+        formatted_examples = format_remarks(examples, "example", "====")
+
+        formatted_notes + formatted_examples
+      end
+      # rubocop:enable Layout/LineLength
+
+      def format_remarks(remarks, remark_item_name, remark_item_symbol)
+        return "" if remarks.nil?
+
+        remarks.map do |remark|
+          <<~REMARK
+
+            [#{remark_item_name}]
+            #{remark_item_symbol}
+            #{remark}
+            #{remark_item_symbol}
+          REMARK
+        end.join
+      end
     end
   end
 end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.16".freeze
+    VERSION = "0.3.19".freeze
   end
 end

data/stepmod-utils.gemspec

@@ -32,7 +32,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "expressir"
   spec.add_runtime_dependency "glossarist", "~> 0.1.0"
+  spec.add_runtime_dependency "indefinite_article"
   spec.add_runtime_dependency "ptools"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.3.5"
   spec.add_runtime_dependency "thor", ">= 0.20.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.16
+  version: 0.3.19
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: expressir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: glossarist
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: indefinite_article
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ptools
   requirement: !ruby/object:Gem::Requirement
@@ -178,7 +206,9 @@ files:
 - lib/stepmod/utils/converters/em_express_description.rb
 - lib/stepmod/utils/converters/eqn.rb
 - lib/stepmod/utils/converters/example.rb
+- lib/stepmod/utils/converters/express_example.rb
 - lib/stepmod/utils/converters/express_g.rb
+- lib/stepmod/utils/converters/express_note.rb
 - lib/stepmod/utils/converters/express_ref.rb
 - lib/stepmod/utils/converters/express_ref_express_description.rb
 - lib/stepmod/utils/converters/ext_description.rb
@@ -190,6 +220,7 @@ files:
 - lib/stepmod/utils/converters/ignore.rb
 - lib/stepmod/utils/converters/introduction.rb
 - lib/stepmod/utils/converters/module_ref.rb
+- lib/stepmod/utils/converters/module_ref_express_description.rb
 - lib/stepmod/utils/converters/note.rb
 - lib/stepmod/utils/converters/ol.rb
 - lib/stepmod/utils/converters/p.rb
@@ -237,7 +268,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.