stepmod-utils 0.3.15 → 0.3.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0fb38c39db845a0b6ff3358d7dea3b0b5f687c50109ee6318438a7787453e5e
-  data.tar.gz: 7ac3bdf768eeac0805f19fd41703632ed12460c4216b98c5a4b689dc99491033
+  metadata.gz: 7d666c11f9282db59fefb04c07c40a88f6167cd214d316f2b1a14ae7117ce898
+  data.tar.gz: fe07f66170d74a97a95003b0f5e3594c5f2bbeaa3d431b716ce52f01d13d418d
 SHA512:
-  metadata.gz: 799b824cd2eec2efe7d3c5a7972ac21ead6af9b493e13e7ec78a3cfb6417e3576835e5aead8ac07ca264ae3513758cb212259d7ed0e0cb9713acb025393c26e6
-  data.tar.gz: 2b95f4f76fb19b09695d6cceddce5d854a4843bdfab62d645363b9fa379ad8616c8e1c361802264703c00684b9a3f7242c0aa22b6448ec8cc2e555cea85e797e
+  metadata.gz: 7b2fea45d8a30e633e393afac572894481d7de3bf5a33cd19043b6b55cb5b2c948666eb705b4cc9e69a12fe7b43217ca80f8f9a47d7aab4507b1b00c4136476e
+  data.tar.gz: 8e5334b99ece0ad5bff37d8ffd7cd70c64fd4998c3e0f4da58825802420f8537493a7858a6158b00ffcfdb3907a532e44067ab4ba07f7cc8f6091b5bb7912517

data/.github/workflows/rake.yml

@@ -10,23 +10,6 @@ on:
 
 jobs:
   rake:
-    name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: [ '3.0', '2.7', '2.6', '2.5' ]
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
-        experimental: [ false ]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-
-      - run: bundle exec rake
+    uses: metanorma/ci/.github/workflows/generic-rake.yml@main
+    secrets:
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/.github/workflows/release.yml

@@ -15,32 +15,10 @@ on:
 
 jobs:
   release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
+    uses: metanorma/ci/.github/workflows/rubygems-release.yml@main
+    with:
+      next_version: ${{ github.event.inputs.next_version }}
+    secrets:
+      rubygems-api-key: ${{ secrets.METANORMA_CI_RUBYGEMS_API_KEY }}
+      pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
 
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: '2.6'
-          bundler-cache: true
-
-      - run: gem install gem-release
-
-      - run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
-
-      - if: github.event_name == 'workflow_dispatch' && github.event.inputs.next_version != 'skip'
-        run: gem bump --version ${{ github.event.inputs.next_version }} --tag --push
-
-      - name: publish to rubygems.org
-        env:
-          RUBYGEMS_API_KEY: ${{secrets.METANORMA_CI_RUBYGEMS_API_KEY}}
-        run: |
-          gem install gem-release
-          envsubst << 'EOF' > ~/.gem/credentials
-          ---
-          :rubygems_api_key: ${RUBYGEMS_API_KEY}
-          EOF
-          chmod 0600 ~/.gem/credentials
-          gem release

data/README.adoc

@@ -14,11 +14,6 @@ The `stepmod-utils` Ruby gem provides a number of tools to work with the STEPmod
 repository.
 
 
-== CVS to Git migration procedures
-
-include::migrating_from_cvs.adoc[]
-
-
 == Installation
 
 [source,ruby]
@@ -116,48 +111,15 @@ expg_image:contract_schemaexpg1.xml[]
 expg_image:contract_schemaexpg2.xml[]
 ----
 
+== Git repository
 
-== CVS revision detection
-
-If the STEPmod repository is a CVS repository, it will display the
-working and repository revisions as comments.
-
-WARNING: CVS revision detection can be *slow*! Be patient.
-
-Sample output:
-
-[source,sh]
-----
-[stepmod-utils] INFO: STEPmod directory set to ../x/stepmod/.
-[stepmod-utils] INFO: STEPmod directory is a CVS repository and will detect revisions.
-[stepmod-utils] INFO: [CVS] Detecting file revisions can be slow, please be patient!
-[stepmod-utils] INFO: Detecting paths...
-[stepmod-utils] INFO: Processing XML file data/business_object_models/managed_model_based_3d_engineering/business_object_model.xml
-[stepmod-utils] INFO: Detecting CVS revision...
-[stepmod-utils] INFO: CVS working rev (1.148), repo rev (1.148)
-[stepmod-utils] INFO: Completed processing XML file data/business_object_models/managed_model_based_3d_engineering/business_object_model.xml
-[stepmod-utils] INFO: Processing XML file data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
-[stepmod-utils] INFO: Detecting CVS revision...
-[stepmod-utils] INFO: CVS working rev (1.31), repo rev (1.31)
-[stepmod-utils] INFO: Completed processing XML file data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
-...
- stepmod/data/business_object_models/managed_model_based_3d_engineering_domain/business_object_model.xml
-[stepmod-utils] INFO: written to: 031-generated-terms.adoc
-[stepmod-utils] INFO: written to: 991-generated-bibliography.adoc
-----
-
-
-== Non-CVS repository
-
-If the STEPmod repository is not a CVS repository, or if the `cvs` executable
-cannot be found, term extract will still work.
+If the STEPmod repository is a Git repository, term extract will work.
 
 Sample output:
 
 [source,sh]
 ----
 [stepmod-utils] INFO: STEPmod directory set to ../iso-10303-stepmod.
-[stepmod-utils] INFO: STEPmod directory is not a CVS repository, skipping revision detection.
 [stepmod-utils] INFO: Detecting paths...
 [stepmod-utils] INFO: Processing XML file data/application_protocols/boundary_representation_for_iso_14306_open_jt/application_protocol.xml
 [stepmod-utils] INFO: skipped ISO/CD 10303-243 as it is not IS, DIS or TS

data/exe/stepmod-annotate-all

@@ -27,15 +27,20 @@ files.each_slice(MAX_QUEUE_SIZE) do |batch|
   batch.each do |file|
     pool.post do
       puts("#{Thread.current.object_id}: Queued processing #{file}")
-      annotated = Stepmod::Utils::StepmodFileAnnotator.new(express_file: file,
-                                                           resource_docs_cache_file: resource_docs_cache_file, stepmod_dir: stepmod_dir).call
-      File.open(
-        File.join(File.dirname(file),
-                  "#{File.basename(file, '.exp')}_annotated.exp"), "w"
-      ) do |file|
+      annotated = Stepmod::Utils::StepmodFileAnnotator.new(
+        express_file: file,
+        resource_docs_cache_file: resource_docs_cache_file,
+        stepmod_dir: stepmod_dir
+      ).call
+
+      annotated_file_name = "#{File.basename(file, '.exp')}_annotated.exp"
+      annotated_file_path = File.join(File.dirname(file), annotated_file_name)
+
+      File.open(annotated_file_path, "w") do |file|
         file.puts(annotated)
       end
-      puts("#{Thread.current.object_id}: Done processing #{file}")
+
+      puts("#{Thread.current.object_id}: Done processing #{File.basename(file)} => #{annotated_file_path}.")
     end
   end
   pool.shutdown

data/exe/stepmod-extract-terms

@@ -30,7 +30,7 @@ options = {}
 OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options]"
 
-  opts.on("-p", "--path STEPMOD_DATA_PATH", String, "Path to STEPmod CVS data directory") do |path|
+  opts.on("-p", "--path STEPMOD_DATA_PATH", String, "Path to STEPmod data directory") do |path|
     options[:stepmod_dir] = path
   end
 
@@ -106,28 +106,29 @@ def replace_images(content)
   content
 end
 
-part_concepts.each do |(bibdata, current_part_concepts)|
-  current_part_concepts = current_part_concepts.to_a.map do |n|
-    n.localizations["en"]
-  end
-  fn = "03x-stepmod-#{bibdata.part}.adoc"
-  File.open(fn, "w") do |file|
-    file.puts("== #{part_to_title(bibdata)}\n\n")
-    file.puts(replace_images(current_part_concepts.map(&:to_mn_adoc).join("\n")))
-  end
-  log "INFO: written to: #{fn}"
-end
-
-File.open("03x-stepmod.adoc", "w") do |file|
-  part_concepts.sort_by do |(bibdata, current_part_concepts)|
-    bibdata.part.to_i
-  end.each do |(bibdata, current_part_concepts)|
-    fn = "03x-stepmod-#{bibdata.part}.adoc"
-    file.puts("\ninclude::#{fn}[]\n")
-  end
-end
-
-log "INFO: written summary file to: 03x-stepmod.adoc"
+## Skip all Clause 3 terms
+# part_concepts.each do |(bibdata, current_part_concepts)|
+#   current_part_concepts = current_part_concepts.to_a.map do |n|
+#     n.localizations["en"]
+#   end
+#   fn = "03x-stepmod-#{bibdata.part}.adoc"
+#   File.open(fn, "w") do |file|
+#     file.puts("== #{part_to_title(bibdata)}\n\n")
+#     file.puts(replace_images(current_part_concepts.map(&:to_mn_adoc).join("\n")))
+#   end
+#   log "INFO: written to: #{fn}"
+# end
+
+# File.open("03x-stepmod.adoc", "w") do |file|
+#   part_concepts.sort_by do |(bibdata, current_part_concepts)|
+#     bibdata.part.to_i
+#   end.each do |(bibdata, current_part_concepts)|
+#     fn = "03x-stepmod-#{bibdata.part}.adoc"
+#     file.puts("\ninclude::#{fn}[]\n")
+#   end
+# end
+
+# log "INFO: written summary file to: 03x-stepmod.adoc"
 
 part_resources.each do |(bibdata, current_part_resources)|
   current_part_resources = current_part_resources.to_a.map do |n|
@@ -194,17 +195,18 @@ end
 
 log "INFO: written summary file to: 05x-stepmod-entities-modules.adoc"
 
-File.open("031-stepmod-general.adoc", "w") do |file|
-  file.puts(
-    replace_images(
-      general_concepts.to_a.map do |n|
-        n.localizations["en"]
-      end.map(&:to_mn_adoc).join("\n")
-    )
-  )
-end
-
-log "INFO: written to: 031-stepmod-general.adoc"
+## Skipping all Clause 3 terms
+# File.open("031-stepmod-general.adoc", "w") do |file|
+#   file.puts(
+#     replace_images(
+#       general_concepts.to_a.map do |n|
+#         n.localizations["en"]
+#       end.map(&:to_mn_adoc).join("\n")
+#     )
+#   )
+# end
+
+# log "INFO: written to: 031-stepmod-general.adoc"
 
 File.open("041-stepmod-entities-resources.adoc", "w") do |file|
   file.puts(

data/lib/stepmod/utils/concept.rb

@@ -23,15 +23,6 @@ module Stepmod
 
           return nil if converted_definition.nil? || converted_definition.strip.empty?
 
-          if definition_xml.name == "ext_description"
-            converted_definition = <<~TEXT
-              #{converted_definition}
-
-              NOTE: This term is incompletely defined in this document.
-              Reference <<#{reference_anchor}>> for the complete definition.
-            TEXT
-          end
-
           # https://github.com/metanorma/stepmod-utils/issues/86
 
           # TODO: This portion DOES NOT HANDLE the <synonym> element. WTF??

data/lib/stepmod/utils/converters/express_example.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class ExpressExample < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          <<~TEMPLATE
+            (*"#{state[:schema_and_entity]}.__example"
+            #{treat_children(node, state).strip}
+            *)
+          TEMPLATE
+        end
+      end
+
+      ReverseAdoc::Converters.register :express_example, ExpressExample.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/express_note.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Stepmod
+  module Utils
+    module Converters
+      class ExpressNote < ReverseAdoc::Converters::Base
+        def convert(node, state = {})
+          <<~TEMPLATE
+            (*"#{state[:schema_and_entity]}.__note"
+            #{treat_children(node, state).strip}
+            *)
+          TEMPLATE
+        end
+      end
+
+      ReverseAdoc::Converters.register :express_note, ExpressNote.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/module_ref_express_description.rb

@@ -0,0 +1,14 @@
+module Stepmod
+  module Utils
+    module Converters
+      class ModuleRefExpressDescription < ReverseAdoc::Converters::Base
+        def convert(node, _state = {})
+          parts = node["linkend"].to_s.split(":").last.split(".")
+          "<<express:#{parts.join('.').strip},#{parts.last.strip}>>"
+        end
+      end
+      ReverseAdoc::Converters.register :module_ref,
+                                       ModuleRefExpressDescription.new
+    end
+  end
+end

data/lib/stepmod/utils/converters/stepmod_ext_description.rb

@@ -43,18 +43,17 @@ module Stepmod
 
           domain =  case linkend.first
                     when /_mim$/, /_arm$/
-                      "ISO 10303 application module"
+                      "application module"
                     # when /_schema$/
                     else
-                      "ISO 10303 resource"
+                      "resource"
                     end
 
           <<~TEMPLATE
             === #{linkend.last}
+            #{domain ? "domain:[#{domain}: #{linkend.first}]" : ''}
 
-            #{domain ? "domain:[#{domain}]" : ''}
-
-            #{child_text}
+            #{child_text.strip}
           TEMPLATE
         end
       end

data/lib/stepmod/utils/converters/term.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "stepmod/utils/converters/synonym"
+require "stepmod/utils/term"
+require "glossarist"
 
 module Stepmod
   module Utils
@@ -22,22 +24,20 @@ module Stepmod
           unless first_child &&
               node.text.split(";").length == 2 &&
               defined?(Stepmod::Utils::Converters::Synonym)
-            return "=== #{treat_acronym(treat_children(node, state).strip)}"
+
+            return Stepmod::Utils::Term.from_h(
+              "definition" => treat_children(node, state).strip,
+            ).to_mn_adoc
           end
 
           term_def, alt = node.text.split(";")
           alt_xml = Nokogiri::XML::Text.new(alt, Nokogiri::XML::Document.new)
           converted_alt = Stepmod::Utils::Converters::Synonym.new.convert(alt_xml)
-          "=== #{treat_acronym(term_def)}\n\n#{converted_alt}"
-        end
-
-        private
-
-        def treat_acronym(term_def)
-          return term_def.strip if term_def !~ /.+\(.+?\)$/
 
-          _, term_text, term_acronym = term_def.match(/(.+?)(\(.+\))$/).to_a
-          "#{term_text.strip}\nalt:[#{term_acronym.gsub(/\(|\)/, '')}]"
+          Stepmod::Utils::Term.from_h(
+            "definition" => term_def,
+            "synonyms" => [converted_alt],
+          ).to_mn_adoc
         end
       end
 

data/lib/stepmod/utils/smrl_description_converter.rb

@@ -41,7 +41,7 @@ module Stepmod
         root || (return "")
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
           Stepmod::Utils::Cleaner.new.tidy(result)
         end
       end

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -59,7 +59,8 @@ module Stepmod
         root || (return "")
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root)
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
+
           Stepmod::Utils::Cleaner.new.tidy(result)
         end
       end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -1,6 +1,8 @@
 require "json"
 require "stepmod/utils/smrl_description_converter"
 require "stepmod/utils/smrl_resource_converter"
+require "stepmod/utils/converters/express_note"
+require "stepmod/utils/converters/express_example"
 
 module Stepmod
   module Utils
@@ -64,7 +66,27 @@ module Stepmod
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<ext_descriptions>#{description}</ext_descriptions>"
-          "\n#{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)}"
+          notes = description.xpath("note")
+          examples = description.xpath("example")
+
+          converted_description = <<~DESCRIPTION
+
+            #{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper, no_notes_examples: true)}
+          DESCRIPTION
+
+          converted_examples = examples.map do |example|
+            Stepmod::Utils::Converters::ExpressExample
+              .new
+              .convert(example, schema_and_entity: description["linkend"])
+          end.join
+
+          converted_notes = notes.map do |note|
+            Stepmod::Utils::Converters::ExpressNote
+              .new
+              .convert(note, schema_and_entity: description["linkend"])
+          end.join
+
+          "#{converted_description}#{converted_examples}#{converted_notes}"
         end
       end
 
@@ -77,7 +99,7 @@ module Stepmod
 
         Dir.chdir(File.dirname(descriptions_file)) do
           wrapper = "<resource>#{schema}</resource>"
-          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper)}"
+          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper, no_notes_examples: true)}"
         end
       end
     end

data/lib/stepmod/utils/term.rb

@@ -0,0 +1,38 @@
+require "glossarist"
+
+module Stepmod
+  module Utils
+    class Term < Glossarist::LocalizedConcept
+      # Term acronym
+      attr_accessor :acronym
+
+      def to_mn_adoc
+        mn_adoc = ["=== #{definition}"]
+        mn_adoc << "\nalt:[#{acronym}]" if acronym
+        mn_adoc << "\n\n#{designations.join(", ")}" if designations&.any?
+
+        mn_adoc.join
+      end
+
+      class << self
+        def from_h(hash)
+          _, definition, acronym = treat_acronym(hash["definition"])
+
+          hash["definition"] = definition
+          hash["acronym"] = acronym.gsub(/\(|\)/, "") if acronym
+          hash["designations"] = hash["synonyms"]
+
+          super(hash.reject { |k, _| k == "synonyms" })
+        end
+
+        private
+
+        def treat_acronym(term_def)
+          return [nil, term_def.strip, nil] unless term_def.match?(/.+\(.+?\)$/)
+
+          term_def.match(/(.+?)(\(.+\))$/).to_a
+        end
+      end
+    end
+  end
+end

data/lib/stepmod/utils/terms_extractor.rb

@@ -3,6 +3,9 @@ require "stepmod/utils/bibdata"
 require "stepmod/utils/concept"
 require "glossarist"
 require "securerandom"
+require "expressir"
+require "expressir/express/parser"
+require "indefinite_article"
 
 ReverseAdoc.config.unknown_tags = :bypass
 
@@ -20,7 +23,6 @@ module Stepmod
                   :resource_concepts,
                   :parsed_bibliography,
                   :encountered_terms,
-                  :cvs_mode,
                   :part_concepts,
                   :part_resources,
                   :part_modules,
@@ -60,25 +62,7 @@ module Stepmod
       end
 
       def call
-        # If we are using the stepmod CVS repository, provide the revision number per file
-        @cvs_mode = if Dir.exists?(stepmod_path.join("CVS"))
-                      require "ptools"
-                      # ptools provides File.which
-                      File.which("cvs")
-                    end
-
         log "INFO: STEPmod directory set to #{stepmod_dir}."
-
-        if cvs_mode
-          log "INFO: STEPmod directory is a \
-            CVS repository and will detect revisions."
-          log "INFO: [CVS] Detecting file revisions can be slow, \
-            please be patient!"
-        else
-          log "INFO: STEPmod directory is not a CVS repository, \
-            skipping revision detection."
-        end
-
         log "INFO: Detecting paths..."
 
         repo_index = Nokogiri::XML(File.read(@index_path)).root
@@ -161,24 +145,15 @@ module Stepmod
             next
           end
 
-          revision_string = "\n// CVS: revision not detected"
-          if cvs_mode
-            # Run `cvs status` to find out version
-
-            log "INFO: Detecting CVS revision..."
-            Dir.chdir(stepmod_path) do
-              status = `cvs status #{fpath}`
-
-              unless status.empty?
-                working_rev = status.split(/\n/).grep(/Working revision:/)
-                  .first.match(/revision:\s+(.+)$/)[1]
-                repo_rev = status.split(/\n/).grep(/Repository revision:/)
-                  .first.match(/revision:\t(.+)\t/)[1]
-                log "INFO: CVS working rev (#{working_rev}), \
-                  repo rev (#{repo_rev})"
-                revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
-                  "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
-              end
+          revision_string = ""
+
+          # Run `cvs status` to find out version
+          log "INFO: Detecting Git SHA..."
+          Dir.chdir(stepmod_path) do
+            git_sha = `git rev-parse HEAD`
+
+            unless git_sha.empty?
+              revision_string = "\n// Git: SHA #{git_sha}"
             end
           end
 
@@ -226,7 +201,7 @@ module Stepmod
           case file_path.to_s
           when /resource.xml$/
             log "INFO: Processing resource.xml for #{file_path}"
-            # Assumption: every schema is only linked by a single resource_docs document.
+
             current_document.xpath("//schema").each do |schema_node|
               schema_name = schema_node["name"]
               if parsed_schema_names[schema_name]
@@ -238,19 +213,36 @@ module Stepmod
                 parsed_schema_names[schema_name] = file_path
               end
 
-              Dir["#{stepmod_path}/resources/#{schema_name}/descriptions.xml"].each do |description_xml_path|
-                log "INFO: Processing resources schema #{description_xml_path}"
-                description_document = Nokogiri::XML(File.read(description_xml_path)).root
-                description_document.xpath("//ext_description").each do |ext_description|
-                  # log "INFO: Processing linkend[#{ext_description['linkend']}]"
-
-                  concept = Stepmod::Utils::Concept.parse(
-                    ext_description,
-                    reference_anchor: bibdata.anchor,
-                    reference_clause: nil,
-                    file_path: Pathname.new(description_xml_path)
+              exp_annotated_path =
+                "#{stepmod_path}/resources/#{schema_name}/#{schema_name}_annotated.exp"
+
+              log "INFO: Processing resources schema #{exp_annotated_path}"
+
+              if File.exists?(exp_annotated_path)
+                repo = Expressir::Express::Parser.from_file(exp_annotated_path)
+                schema = repo.schemas.first
+
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  domain = "resource: #{schema.id}"
+                  entity_definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: entity_definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
                                 .relative_path_from(stepmod_path),
+                    language_code: "en",
                   )
+
                   next unless concept
 
                   if term_special_category(bibdata)
@@ -280,62 +272,89 @@ module Stepmod
               parsed_schema_names[schema_name] = file_path
             end
 
-            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/arm_descriptions.xml"
-            log "INFO: Processing modules schema #{description_xml_path}"
-
-            if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(
-                File.read(description_xml_path),
-              )
-                .root
-              description_document.xpath("//ext_description").each do |ext_description|
-                linkend_schema = ext_description["linkend"].split(".").first
-                concept = Stepmod::Utils::Concept.parse(
-                  ext_description,
-                  reference_anchor: bibdata.anchor,
-                  reference_clause: nil,
-                  file_path: Pathname.new(description_xml_path)
-                              .relative_path_from(stepmod_path),
-                )
-                next unless concept
-
-                current_part_modules_arm[linkend_schema] ||= Glossarist::Collection.new
-                find_or_initialize_concept(
-                  current_part_modules_arm[linkend_schema], concept
-                )
-                # puts part_modules_arm.inspect
-                parsed_bibliography << bibdata
+            exp_annotated_path =
+              "#{stepmod_path}/modules/#{schema_name}/arm_annotated.exp"
+
+            log "INFO: Processing modules schema #{exp_annotated_path}"
+
+            if File.exists?(exp_annotated_path)
+              repo = Expressir::Express::Parser.from_file(exp_annotated_path)
+
+              repo.schemas.each do |schema|
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  domain = "application module: #{schema.id}"
+                  entity_definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: entity_definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
+                                .relative_path_from(stepmod_path),
+                    language_code: "en",
+                  )
+
+                  next unless concept
+
+                  current_part_modules_arm[schema.id] ||=
+                    Glossarist::Collection.new
+                  find_or_initialize_concept(
+                    current_part_modules_arm[schema.id], concept
+                  )
+
+                  # puts part_modules_arm.inspect
+                  parsed_bibliography << bibdata
+                end
               end
             end
 
-            description_xml_path = "#{stepmod_path}/modules/#{schema_name}/mim_descriptions.xml"
-            log "INFO: Processing modules schema #{description_xml_path}"
-
-            if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(
-                File.read(description_xml_path),
-              )
-                .root
-              description_document.xpath("//ext_description").each do |ext_description|
-                linkend_schema = ext_description["linkend"].split(".").first
-
-                concept = Stepmod::Utils::Concept.parse(
-                  ext_description,
-                  reference_anchor: bibdata.anchor,
-                  reference_clause: nil,
-                  file_path: Pathname
-                              .new(description_xml_path)
-                              .relative_path_from(stepmod_path),
-                )
-                next unless concept
-
-                current_part_modules_mim[linkend_schema] ||=
-                  Glossarist::Collection.new
-                find_or_initialize_concept(
-                  current_part_modules_mim[linkend_schema], concept
-                )
-
-                parsed_bibliography << bibdata
+            mim_exp_annotated_path = "#{stepmod_path}/modules/#{schema_name}/mim_annotated.exp"
+
+            log "INFO: Processing modules schema #{mim_exp_annotated_path}"
+
+            if File.exists?(mim_exp_annotated_path)
+              repo = Expressir::Express::Parser.from_file(mim_exp_annotated_path)
+
+              repo.schemas.each do |schema|
+                schema.entities.each do |entity|
+                  old_definition = entity.remarks.first
+
+                  domain = "application module: #{schema.id}"
+                  definition = generate_entity_definition(entity, domain, old_definition)
+
+                  reference_anchor = bibdata.anchor
+                  reference_clause = nil
+
+                  concept = Stepmod::Utils::Concept.new(
+                    designations: [entity.id],
+                    definition: old_definition,
+                    converted_definition: definition,
+                    id: "#{reference_anchor}.#{reference_clause}",
+                    reference_anchor: reference_anchor,
+                    reference_clause: reference_clause,
+                    file_path: Pathname.new(exp_annotated_path)
+                                .relative_path_from(stepmod_path),
+                    language_code: "en",
+                  )
+
+                  next unless concept
+
+                  current_part_modules_mim[schema.id] ||=
+                    Glossarist::Collection.new
+                  find_or_initialize_concept(
+                    current_part_modules_mim[schema.id], concept
+                  )
+
+                  parsed_bibliography << bibdata
+                end
               end
             end
 
@@ -371,10 +390,150 @@ module Stepmod
       end
 
       def find_or_initialize_concept(collection, localized_concept)
+        concept_id = SecureRandom.uuid
+
         concept = collection
-          .store(Glossarist::Concept.new(id: SecureRandom.uuid))
+          .store(Glossarist::Concept.new(id: concept_id))
         concept.add_l10n(localized_concept)
       end
+
+      def combine_paragraphs(full_paragraph, next_paragraph)
+        # If full_paragraph already contains a period, extract that.
+        if m = full_paragraph.match(/\A(?<inner_first>[^\n]*?\.)\s/)
+          # puts "CONDITION 1"
+          if m[:inner_first]
+            return m[:inner_first]
+          else
+            return full_paragraph
+          end
+        end
+
+        # If full_paragraph ends with a period, this is the last.
+        if full_paragraph =~ /\.\s*\Z/
+          # puts "CONDITION 2"
+          return full_paragraph
+        end
+
+        # If next_paragraph is a list
+        if next_paragraph.match(/\A\*/)
+          # puts "CONDITION 3"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # If next_paragraph is a continuation of a list
+        if next_paragraph.match(/\Awhich/) || next_paragraph.match(/\Athat/)
+          # puts "CONDITION 4"
+          return full_paragraph + "\n\n" + next_paragraph
+        end
+
+        # puts "CONDITION 5"
+        full_paragraph
+      end
+
+      def trim_definition(definition)
+        # Unless the first paragraph ends with "between" and is followed by a
+        # list, don't split
+        paragraphs = definition.split("\n\n")
+
+        # puts paragraphs.inspect
+
+        first_paragraph = paragraphs.first
+
+        if paragraphs.length > 1
+          combined = paragraphs[1..-1].inject(first_paragraph) do |acc, p|
+            combine_paragraphs(acc, p)
+          end
+        else
+          combined = combine_paragraphs(first_paragraph, "")
+        end
+
+        # puts "combined--------- #{combined}"
+
+        # Remove comments until end of line
+        combined = combined + "\n"
+        combined.gsub!(/\n\/\/.*?\n/, "\n")
+        combined.strip!
+
+        combined
+        # # TODO: If the definition contains a list immediately after the first paragraph, don't split
+        # return definition if definition =~ /\n\* /
+
+        # unless (
+        #   first_paragraph =~ /between:?\s*\Z/ ||
+        #   first_paragraph =~ /include:?\s*\Z/ ||
+        #   first_paragraph =~ /of:?\s*\Z/ ||
+        #   first_paragraph =~ /[:;]\s*\Z/
+        #   ) &&
+        #   definition =~ /\n\n\*/
+
+        #   # Only taking the first paragraph of the definition
+        #   first_paragraph
+        # end
+      end
+
+
+      # rubocop:disable Layout/LineLength
+      def generate_entity_definition(entity, domain, old_definition)
+        return "" if entity.nil?
+
+        entity_text = if entity.subtype_of.size.zero?
+                        "entity data type that represents " +
+                        entity.id.indefinite_article + " **#{entity.id}** entity"
+                      else
+                        "entity data type that is a type of "+
+                        "**#{entity.subtype_of.map(&:id).join('** and **')}** that represents " +
+                        entity.id.indefinite_article + " **#{entity.id}** entity"
+                      end
+
+        definition = <<~DEFINITION
+          === #{entity.id}
+          domain:[#{domain}]
+
+          #{entity_text}
+
+        DEFINITION
+
+        unless old_definition.nil? || old_definition.blank?
+          old_definition = trim_definition(old_definition)
+
+          definition << <<~OLD_DEFINITION
+            [NOTE]
+            --
+            #{old_definition.strip}
+            --
+          OLD_DEFINITION
+        end
+
+        # We no longer add Notes and Examples to the extracted terms
+        # definition + format_remark_items(entity.remark_items)
+
+        definition
+      end
+
+      def format_remark_items(remark_items)
+        notes = remark_items.detect { |i| i.id == "__note" }&.remarks
+        examples = remark_items.detect { |i| i.id == "__example" }&.remarks
+
+        formatted_notes = format_remarks(notes, "NOTE", "--")
+        formatted_examples = format_remarks(examples, "example", "====")
+
+        formatted_notes + formatted_examples
+      end
+      # rubocop:enable Layout/LineLength
+
+      def format_remarks(remarks, remark_item_name, remark_item_symbol)
+        return "" if remarks.nil?
+
+        remarks.map do |remark|
+          <<~REMARK
+
+            [#{remark_item_name}]
+            #{remark_item_symbol}
+            #{remark}
+            #{remark_item_symbol}
+          REMARK
+        end.join
+      end
     end
   end
 end

data/lib/stepmod/utils/version.rb

@@ -1,5 +1,5 @@
 module Stepmod
   module Utils
-    VERSION = "0.3.15".freeze
+    VERSION = "0.3.18".freeze
   end
 end

data/stepmod-utils.gemspec

@@ -32,7 +32,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "expressir"
   spec.add_runtime_dependency "glossarist", "~> 0.1.0"
+  spec.add_runtime_dependency "indefinite_article"
   spec.add_runtime_dependency "ptools"
   spec.add_runtime_dependency "reverse_adoc", ">= 0.3.5"
   spec.add_runtime_dependency "thor", ">= 0.20.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stepmod-utils
 version: !ruby/object:Gem::Version
-  version: 0.3.15
+  version: 0.3.18
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-16 00:00:00.000000000 Z
+date: 2022-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: expressir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: glossarist
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: indefinite_article
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ptools
   requirement: !ruby/object:Gem::Requirement
@@ -178,7 +206,9 @@ files:
 - lib/stepmod/utils/converters/em_express_description.rb
 - lib/stepmod/utils/converters/eqn.rb
 - lib/stepmod/utils/converters/example.rb
+- lib/stepmod/utils/converters/express_example.rb
 - lib/stepmod/utils/converters/express_g.rb
+- lib/stepmod/utils/converters/express_note.rb
 - lib/stepmod/utils/converters/express_ref.rb
 - lib/stepmod/utils/converters/express_ref_express_description.rb
 - lib/stepmod/utils/converters/ext_description.rb
@@ -190,6 +220,7 @@ files:
 - lib/stepmod/utils/converters/ignore.rb
 - lib/stepmod/utils/converters/introduction.rb
 - lib/stepmod/utils/converters/module_ref.rb
+- lib/stepmod/utils/converters/module_ref_express_description.rb
 - lib/stepmod/utils/converters/note.rb
 - lib/stepmod/utils/converters/ol.rb
 - lib/stepmod/utils/converters/p.rb
@@ -213,6 +244,7 @@ files:
 - lib/stepmod/utils/smrl_resource_converter.rb
 - lib/stepmod/utils/stepmod_definition_converter.rb
 - lib/stepmod/utils/stepmod_file_annotator.rb
+- lib/stepmod/utils/term.rb
 - lib/stepmod/utils/terms_extractor.rb
 - lib/stepmod/utils/version.rb
 - resource_example.xml
@@ -236,7 +268,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Stepmod-utils is a toolkit that works on STEPmod data.