stepmod-utils 0.3.5 → 0.3.7

data/lib/stepmod/utils/smrl_description_converter.rb

@@ -1,43 +1,44 @@
 # frozen_string_literal: true
 
-require 'reverse_adoc'
-require 'stepmod/utils/converters/a'
-require 'stepmod/utils/converters/blockquote'
-require 'stepmod/utils/converters/br'
-require 'stepmod/utils/converters/bypass'
-require 'stepmod/utils/converters/code'
-require 'stepmod/utils/converters/drop'
-require 'stepmod/utils/converters/em_express_description'
-require 'stepmod/utils/converters/example'
-require 'stepmod/utils/converters/express_ref_express_description'
-require 'stepmod/utils/converters/ext_description'
-require 'stepmod/utils/converters/ext_descriptions'
-require 'stepmod/utils/converters/head'
-require 'stepmod/utils/converters/hr'
-require 'stepmod/utils/converters/ignore'
-require 'stepmod/utils/converters/note'
-require 'stepmod/utils/converters/p'
-require 'stepmod/utils/converters/pass_through'
-require 'stepmod/utils/converters/q'
-require 'stepmod/utils/converters/strong'
-require 'stepmod/utils/converters/sub'
-require 'stepmod/utils/converters/sup'
-require 'stepmod/utils/converters/text'
-require 'stepmod/utils/cleaner'
+require "reverse_adoc"
+require "stepmod/utils/converters/a"
+require "stepmod/utils/converters/blockquote"
+require "stepmod/utils/converters/br"
+require "stepmod/utils/converters/bypass"
+require "stepmod/utils/converters/code"
+require "stepmod/utils/converters/drop"
+require "stepmod/utils/converters/em_express_description"
+require "stepmod/utils/converters/example"
+require "stepmod/utils/converters/express_ref_express_description"
+require "stepmod/utils/converters/ext_description"
+require "stepmod/utils/converters/ext_descriptions"
+require "stepmod/utils/converters/head"
+require "stepmod/utils/converters/hr"
+require "stepmod/utils/converters/ignore"
+require "stepmod/utils/converters/note"
+require "stepmod/utils/converters/p"
+require "stepmod/utils/converters/pass_through"
+require "stepmod/utils/converters/q"
+require "stepmod/utils/converters/strong"
+require "stepmod/utils/converters/sub"
+require "stepmod/utils/converters/sup"
+require "stepmod/utils/converters/text"
+require "stepmod/utils/cleaner"
 
 module Stepmod
   module Utils
     class SmrlDescriptionConverter
       def self.convert(input, options = {})
-        root = if input.is_a?(String)
-                  then Nokogiri::XML(input).root
-                elsif input.is_a?(Nokogiri::XML::Document)
-                  then input.root
-                elsif input.is_a?(Nokogiri::XML::Node)
-                  then input
-                end
+        root = case input
+               when String
+                 Nokogiri::XML(input).root
+               when Nokogiri::XML::Document
+                 input.root
+               when Nokogiri::XML::Node
+                 input
+               end
 
-        root || (return '')
+        root || (return "")
 
         ReverseAdoc.config.with(options) do
           result = ReverseAdoc::Converters.lookup(root.name).convert(root)
@@ -46,4 +47,4 @@ module Stepmod
       end
     end
   end
-end
+end

data/lib/stepmod/utils/smrl_resource_converter.rb

@@ -1,61 +1,62 @@
 # frozen_string_literal: true
 
-require 'reverse_adoc'
-require 'reverse_adoc/converters/bypass'
-require 'reverse_adoc/converters/pass_through'
-require 'stepmod/utils/converters/a'
-require 'stepmod/utils/converters/blockquote'
-require 'stepmod/utils/converters/br'
-require 'stepmod/utils/converters/bypass'
-require 'stepmod/utils/converters/code'
-require 'stepmod/utils/converters/comment'
-require 'stepmod/utils/converters/dd'
-require 'stepmod/utils/converters/dl'
-require 'stepmod/utils/converters/dt'
-require 'stepmod/utils/converters/drop'
-require 'stepmod/utils/converters/example'
-require 'stepmod/utils/converters/express_g'
-require 'stepmod/utils/converters/fund_cons'
-require 'stepmod/utils/converters/eqn'
-require 'stepmod/utils/converters/head'
-require 'stepmod/utils/converters/hr'
-require 'stepmod/utils/converters/ignore'
-require 'stepmod/utils/converters/introduction'
-require 'stepmod/utils/converters/note'
-require 'stepmod/utils/converters/ol'
-require 'stepmod/utils/converters/p'
-require 'stepmod/utils/converters/pass_through'
-require 'stepmod/utils/converters/q'
-require 'stepmod/utils/converters/resource'
-require 'stepmod/utils/converters/schema_diag'
-require 'stepmod/utils/converters/schema'
-require 'stepmod/utils/converters/strong'
-require 'stepmod/utils/converters/sub'
-require 'stepmod/utils/converters/sup'
-require 'stepmod/utils/converters/table'
-require 'stepmod/utils/converters/text'
-require 'stepmod/utils/cleaner'
+require "reverse_adoc"
+require "reverse_adoc/converters/bypass"
+require "reverse_adoc/converters/pass_through"
+require "stepmod/utils/converters/a"
+require "stepmod/utils/converters/blockquote"
+require "stepmod/utils/converters/br"
+require "stepmod/utils/converters/bypass"
+require "stepmod/utils/converters/code"
+require "stepmod/utils/converters/comment"
+require "stepmod/utils/converters/dd"
+require "stepmod/utils/converters/dl"
+require "stepmod/utils/converters/dt"
+require "stepmod/utils/converters/drop"
+require "stepmod/utils/converters/example"
+require "stepmod/utils/converters/express_g"
+require "stepmod/utils/converters/fund_cons"
+require "stepmod/utils/converters/eqn"
+require "stepmod/utils/converters/head"
+require "stepmod/utils/converters/hr"
+require "stepmod/utils/converters/ignore"
+require "stepmod/utils/converters/introduction"
+require "stepmod/utils/converters/note"
+require "stepmod/utils/converters/ol"
+require "stepmod/utils/converters/p"
+require "stepmod/utils/converters/pass_through"
+require "stepmod/utils/converters/q"
+require "stepmod/utils/converters/resource"
+require "stepmod/utils/converters/schema_diag"
+require "stepmod/utils/converters/schema"
+require "stepmod/utils/converters/strong"
+require "stepmod/utils/converters/sub"
+require "stepmod/utils/converters/sup"
+require "stepmod/utils/converters/table"
+require "stepmod/utils/converters/text"
+require "stepmod/utils/cleaner"
 
-require 'reverse_adoc/converters/figure'
-require 'reverse_adoc/converters/img'
-require 'reverse_adoc/converters/li'
-require 'reverse_adoc/converters/tr'
-require 'reverse_adoc/converters/td'
-require 'reverse_adoc/converters/th'
+require "reverse_adoc/converters/figure"
+require "reverse_adoc/converters/img"
+require "reverse_adoc/converters/li"
+require "reverse_adoc/converters/tr"
+require "reverse_adoc/converters/td"
+require "reverse_adoc/converters/th"
 
 module Stepmod
   module Utils
     class SmrlResourceConverter
       def self.convert(input, options = {})
-        root = if input.is_a?(String)
-                  then Nokogiri::XML(input).root
-                elsif input.is_a?(Nokogiri::XML::Document)
-                  then input.root
-                elsif input.is_a?(Nokogiri::XML::Node)
-                  then input
-                end
+        root = case input
+               when String
+                 Nokogiri::XML(input).root
+               when Nokogiri::XML::Document
+                 input.root
+               when Nokogiri::XML::Node
+                 input
+               end
 
-        root || (return '')
+        root || (return "")
 
         ReverseAdoc.config.with(options) do
           result = ReverseAdoc::Converters.lookup(root.name).convert(root)

data/lib/stepmod/utils/stepmod_definition_converter.rb

@@ -1,41 +1,40 @@
 # frozen_string_literal: true
 
-require 'reverse_adoc'
-require 'stepmod/utils/converters/arm'
-require 'stepmod/utils/converters/clause_ref'
-require 'stepmod/utils/converters/express_ref'
-require 'stepmod/utils/converters/module_ref'
-require 'stepmod/utils/converters/def'
-require 'stepmod/utils/converters/definition'
-require 'stepmod/utils/converters/em'
-require 'stepmod/utils/converters/example'
-require 'stepmod/utils/converters/note'
-require 'stepmod/utils/converters/ol'
-require 'stepmod/utils/converters/stem'
-require 'stepmod/utils/converters/stepmod_ext_description'
-require 'stepmod/utils/converters/term'
-require 'stepmod/utils/converters/synonym'
-require 'stepmod/utils/converters/uof'
-require 'stepmod/utils/converters/figure'
-
-require 'reverse_adoc/converters/a'
-require 'reverse_adoc/converters/blockquote'
-require 'reverse_adoc/converters/bypass'
-require 'reverse_adoc/converters/br'
-require 'reverse_adoc/converters/code'
-require 'reverse_adoc/converters/drop'
-require 'reverse_adoc/converters/head'
-require 'reverse_adoc/converters/hr'
-require 'reverse_adoc/converters/ignore'
-require 'reverse_adoc/converters/li'
-require 'reverse_adoc/converters/p'
-require 'reverse_adoc/converters/pass_through'
-require 'reverse_adoc/converters/q'
-require 'reverse_adoc/converters/strong'
-require 'reverse_adoc/converters/sup'
-require 'reverse_adoc/converters/sub'
-require 'reverse_adoc/converters/text'
+require "reverse_adoc"
+require "stepmod/utils/converters/arm"
+require "stepmod/utils/converters/clause_ref"
+require "stepmod/utils/converters/express_ref"
+require "stepmod/utils/converters/module_ref"
+require "stepmod/utils/converters/def"
+require "stepmod/utils/converters/definition"
+require "stepmod/utils/converters/em"
+require "stepmod/utils/converters/example"
+require "stepmod/utils/converters/note"
+require "stepmod/utils/converters/ol"
+require "stepmod/utils/converters/stem"
+require "stepmod/utils/converters/stepmod_ext_description"
+require "stepmod/utils/converters/term"
+require "stepmod/utils/converters/synonym"
+require "stepmod/utils/converters/uof"
+require "stepmod/utils/converters/figure"
 
+require "reverse_adoc/converters/a"
+require "reverse_adoc/converters/blockquote"
+require "reverse_adoc/converters/bypass"
+require "reverse_adoc/converters/br"
+require "reverse_adoc/converters/code"
+require "reverse_adoc/converters/drop"
+require "reverse_adoc/converters/head"
+require "reverse_adoc/converters/hr"
+require "reverse_adoc/converters/ignore"
+require "reverse_adoc/converters/li"
+require "reverse_adoc/converters/p"
+require "reverse_adoc/converters/pass_through"
+require "reverse_adoc/converters/q"
+require "reverse_adoc/converters/strong"
+require "reverse_adoc/converters/sup"
+require "reverse_adoc/converters/sub"
+require "reverse_adoc/converters/text"
 
 module Stepmod
   module Utils
@@ -50,11 +49,13 @@ module Stepmod
                  input
                end
 
-        return '' unless root
+        return "" unless root
 
         ReverseAdoc.config.with(options) do
-          result = ReverseAdoc::Converters.lookup(root.name).convert(root, options)
-          return '' unless result
+          result = ReverseAdoc::Converters.lookup(root.name).convert(root,
+                                                                     options)
+          return "" unless result
+
           ReverseAdoc.cleaner.tidy(result.dup)
         end
       end

data/lib/stepmod/utils/stepmod_file_annotator.rb

@@ -1,6 +1,6 @@
-require 'json'
-require 'stepmod/utils/smrl_description_converter'
-require 'stepmod/utils/smrl_resource_converter'
+require "json"
+require "stepmod/utils/smrl_description_converter"
+require "stepmod/utils/smrl_resource_converter"
 
 module Stepmod
   module Utils
@@ -17,35 +17,41 @@ module Stepmod
 
       def call
         match = File.basename(express_file).match('^(arm|mim|bom)\.exp$')
-        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : 'descriptions.xml'
-        descriptions_file = File.join(File.dirname(express_file), descriptions_base)
+        descriptions_base = match ? "#{match.captures[0]}_descriptions.xml" : "descriptions.xml"
+        descriptions_file = File.join(File.dirname(express_file),
+                                      descriptions_base)
         output_express = File.read(express_file)
         resource_docs_cache = JSON.parse(File.read(resource_docs_cache_file))
 
         if File.exists?(descriptions_file)
           descriptions = Nokogiri::XML(File.read(descriptions_file)).root
           added_resource_descriptions = {}
-          descriptions.xpath('ext_description').each do |description|
+          descriptions.xpath("ext_description").each do |description|
             # Add base resource from linked path if exists, eg "language_schema.language.wr:WR1" -> "language_schema"
-            base_linked = description['linkend'].to_s.split('.').first
+            base_linked = description["linkend"].to_s.split(".").first
             if added_resource_descriptions[base_linked].nil?
-              base_reource_doc_dir = resource_docs_cache[description['linkend'].to_s.split('.').first]
+              base_reource_doc_dir = resource_docs_cache[description["linkend"].to_s.split(".").first]
               if base_reource_doc_dir
-                output_express << convert_from_resource_file(base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file)
+                output_express << convert_from_resource_file(
+                  base_reource_doc_dir, stepmod_dir, base_linked, descriptions_file
+                )
               end
               added_resource_descriptions[base_linked] = true
             end
-            resource_docs_dir = resource_docs_cache[description['linkend']]
+            resource_docs_dir = resource_docs_cache[description["linkend"]]
             # Convert content description
             # when a schema description is available from resource.xml and also descriptions.xml, the description from resource.xml is only used.
             # https://github.com/metanorma/annotated-express/issues/32#issuecomment-792609078
             if description.text.strip.length.positive? && resource_docs_dir.nil?
-              output_express << convert_from_description_text(descriptions_file, description)
+              output_express << convert_from_description_text(
+                descriptions_file, description
+              )
             end
             # Add converted description from exact linked path
-            if resource_docs_dir && added_resource_descriptions[description['linkend']].nil?
-              output_express << convert_from_resource_file(resource_docs_dir, stepmod_dir, description['linkend'], descriptions_file)
-              added_resource_descriptions[description['linkend']] = true
+            if resource_docs_dir && added_resource_descriptions[description["linkend"]].nil?
+              output_express << convert_from_resource_file(resource_docs_dir,
+                                                           stepmod_dir, description["linkend"], descriptions_file)
+              added_resource_descriptions[description["linkend"]] = true
             end
           end
         end
@@ -57,20 +63,21 @@ module Stepmod
 
       def convert_from_description_text(descriptions_file, description)
         Dir.chdir(File.dirname(descriptions_file)) do
-          wrapper = "<ext_descriptions>#{description.to_s}</ext_descriptions>"
-          "\n" + Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)
+          wrapper = "<ext_descriptions>#{description}</ext_descriptions>"
+          "\n#{Stepmod::Utils::SmrlDescriptionConverter.convert(wrapper)}"
         end
       end
 
       def convert_from_resource_file(resource_docs_dir, stepmod_dir, linked, descriptions_file)
-        resource_docs_file = File.join(stepmod_dir, 'data/resource_docs', resource_docs_dir, 'resource.xml')
+        resource_docs_file = File.join(stepmod_dir, "data/resource_docs",
+                                       resource_docs_dir, "resource.xml")
         puts(resource_docs_file)
         resource_docs = Nokogiri::XML(File.read(resource_docs_file)).root
         schema = resource_docs.xpath("schema[@name='#{linked}']")
 
         Dir.chdir(File.dirname(descriptions_file)) do
-          wrapper = "<resource>#{schema.to_s}</resource>"
-          "\n" + Stepmod::Utils::SmrlResourceConverter.convert(wrapper)
+          wrapper = "<resource>#{schema}</resource>"
+          "\n#{Stepmod::Utils::SmrlResourceConverter.convert(wrapper)}"
         end
       end
     end

data/lib/stepmod/utils/terms_extractor.rb

@@ -1,6 +1,8 @@
-require 'stepmod/utils/stepmod_definition_converter'
-require 'stepmod/utils/bibdata'
-require 'stepmod/utils/concept'
+require "stepmod/utils/stepmod_definition_converter"
+require "stepmod/utils/bibdata"
+require "stepmod/utils/concept"
+require "glossarist"
+require "securerandom"
 
 ReverseAdoc.config.unknown_tags = :bypass
 
@@ -8,21 +10,22 @@ module Stepmod
   module Utils
     class TermsExtractor
       # TODO: we may want a command line option to override this in the future
-      ACCEPTED_STAGES = %w(IS DIS FDIS TS)
+      ACCEPTED_STAGES = %w(IS DIS FDIS TS).freeze
+      WITHDRAWN_STATUS = "withdrawn".freeze
 
       attr_reader :stepmod_path,
-        :stepmod_dir,
-        :general_concepts,
-        :resource_concepts,
-        :parsed_bibliography,
-        :encountered_terms,
-        :cvs_mode,
-        :part_concepts,
-        :part_resources,
-        :part_modules,
-        :stdout
-
-      def self.call(stepmod_dir, stdout = STDOUT)
+                  :stepmod_dir,
+                  :general_concepts,
+                  :resource_concepts,
+                  :parsed_bibliography,
+                  :encountered_terms,
+                  :cvs_mode,
+                  :part_concepts,
+                  :part_resources,
+                  :part_modules,
+                  :stdout
+
+      def self.call(stepmod_dir, stdout = $stdout)
         new(stepmod_dir, stdout).call
       end
 
@@ -30,8 +33,8 @@ module Stepmod
         @stdout = stdout
         @stepmod_dir = stepmod_dir
         @stepmod_path = Pathname.new(stepmod_dir).realpath
-        @general_concepts = []
-        @resource_concepts = []
+        @general_concepts = Glossarist::Collection.new
+        @resource_concepts = Glossarist::Collection.new
         @parsed_bibliography = []
         @part_concepts = []
         @part_resources = []
@@ -39,13 +42,13 @@ module Stepmod
         @encountered_terms = {}
       end
 
-      def log message
+      def log(message)
         stdout.puts "[stepmod-utils] #{message}"
       end
 
       def term_special_category(bibdata)
         case bibdata.part.to_i
-        when 41,42,43,44,45,46,47,51
+        when 41, 42, 43, 44, 45, 46, 47, 51
           true
         when [56..112]
           true
@@ -56,47 +59,58 @@ module Stepmod
 
       def call
         # If we are using the stepmod CVS repository, provide the revision number per file
-        @cvs_mode = if Dir.exists?(stepmod_path.join('CVS'))
-          require 'ptools'
-          # ptools provides File.which
-          File.which("cvs")
-        end
+        @cvs_mode = if Dir.exists?(stepmod_path.join("CVS"))
+                      require "ptools"
+                      # ptools provides File.which
+                      File.which("cvs")
+                    end
 
         log "INFO: STEPmod directory set to #{stepmod_dir}."
 
         if cvs_mode
-          log "INFO: STEPmod directory is a CVS repository and will detect revisions."
-          log "INFO: [CVS] Detecting file revisions can be slow, please be patient!"
+          log "INFO: STEPmod directory is a \
+            CVS repository and will detect revisions."
+          log "INFO: [CVS] Detecting file revisions can be slow, \
+            please be patient!"
         else
-          log "INFO: STEPmod directory is not a CVS repository, skipping revision detection."
+          log "INFO: STEPmod directory is not a CVS repository, \
+            skipping revision detection."
         end
 
         log "INFO: Detecting paths..."
 
-        repo_index = Nokogiri::XML(File.read(stepmod_path.join('repository_index.xml'))).root
+        repo_index = Nokogiri::XML(File.read(stepmod_path.join("repository_index.xml"))).root
 
         files = []
 
         # add module paths
-        repo_index.xpath('//module').each do |x|
+        repo_index.xpath("//module").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
           path = Pathname.new("#{stepmod_dir}/modules/#{x['name']}/module.xml")
           files << path if File.exists? path
         end
 
         # add resource_docs paths
-        repo_index.xpath('//resource_doc').each do |x|
+        repo_index.xpath("//resource_doc").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
           path = Pathname.new("#{stepmod_dir}/resource_docs/#{x['name']}/resource.xml")
           files << path if File.exists? path
         end
 
         # add business_object_models paths
-        repo_index.xpath('//business_object_model').each do |x|
+        repo_index.xpath("//business_object_model").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
           path = Pathname.new("#{stepmod_dir}/business_object_models/#{x['name']}/business_object_model.xml")
           files << path if File.exists? path
         end
 
         # add application_protocols paths
-        repo_index.xpath('//application_protocol').each do |x|
+        repo_index.xpath("//application_protocol").each do |x|
+          next if x['status'] == WITHDRAWN_STATUS
+
           path = Pathname.new("#{stepmod_dir}/application_protocols/#{x['name']}/application_protocol.xml")
           files << path if File.exists? path
         end
@@ -110,7 +124,7 @@ module Stepmod
           parsed_bibliography,
           part_concepts,
           part_resources,
-          part_modules
+          part_modules,
         ]
       end
 
@@ -128,13 +142,14 @@ module Stepmod
           bibdata = nil
           begin
             bibdata = Stepmod::Utils::Bibdata.new(document: current_document)
-          rescue
+          rescue StandardError
             log "WARNING: Unknown file #{fpath}, skipped"
             next
           end
 
           unless ACCEPTED_STAGES.include? bibdata.doctype
-            log "INFO: skipped #{bibdata.docid} as it is not one of (#{ACCEPTED_STAGES.join(", ")})."
+            log "INFO: skipped #{bibdata.docid} as it is not \
+              one of (#{ACCEPTED_STAGES.join(', ')})."
             next
           end
 
@@ -153,9 +168,12 @@ module Stepmod
               status = `cvs status #{fpath}`
 
               unless status.empty?
-                working_rev = status.split(/\n/).grep(/Working revision:/).first.match(/revision:\s+(.+)$/)[1]
-                repo_rev = status.split(/\n/).grep(/Repository revision:/).first.match(/revision:\t(.+)\t/)[1]
-                log "INFO: CVS working rev (#{working_rev}), repo rev (#{repo_rev})"
+                working_rev = status.split(/\n/).grep(/Working revision:/)
+                  .first.match(/revision:\s+(.+)$/)[1]
+                repo_rev = status.split(/\n/).grep(/Repository revision:/)
+                  .first.match(/revision:\t(.+)\t/)[1]
+                log "INFO: CVS working rev (#{working_rev}), \
+                  repo rev (#{repo_rev})"
                 revision_string = "\n// CVS working rev: (#{working_rev}), repo rev (#{repo_rev})\n" +
                   "// CVS: revision #{working_rev == repo_rev ? 'up to date' : 'differs'}"
               end
@@ -163,11 +181,11 @@ module Stepmod
           end
 
           # read definitions
-          current_part_concepts = []
+          current_part_concepts = Glossarist::Collection.new
           definition_index = 0
-          current_document.xpath('//definition').each do |definition|
+          current_document.xpath("//definition").each do |definition|
             definition_index += 1
-            term_id = definition['id']
+            term_id = definition["id"]
             unless term_id.nil?
               if encountered_terms[term_id]
                 log "FATAL: Duplicated term with id: #{term_id}, #{fpath}"
@@ -183,22 +201,22 @@ module Stepmod
               definition,
               reference_anchor: bibdata.anchor,
               reference_clause: ref_clause,
-              file_path: fpath + revision_string
+              file_path: fpath + revision_string,
             )
             next unless concept
 
-            unless term_special_category(bibdata)
-              # log "INFO: this part is generic"
-              general_concepts << concept
-            else
+            if term_special_category(bibdata)
               # log "INFO: this part is special"
-              current_part_concepts << concept
+              find_or_initialize_concept(current_part_concepts, concept)
+            else
+              # log "INFO: this part is generic"
+              find_or_initialize_concept(general_concepts, concept)
             end
 
             parsed_bibliography << bibdata
           end
 
-          current_part_resources = []
+          current_part_resources = Glossarist::Collection.new
           current_part_modules_arm = {}
           current_part_modules_mim = {}
 
@@ -207,10 +225,12 @@ module Stepmod
           when /resource.xml$/
             log "INFO: Processing resource.xml for #{file_path}"
             # Assumption: every schema is only linked by a single resource_docs document.
-            current_document.xpath('//schema').each do |schema_node|
-              schema_name = schema_node['name']
+            current_document.xpath("//schema").each do |schema_node|
+              schema_name = schema_node["name"]
               if parsed_schema_names[schema_name]
-                log "ERROR: We have encountered this schema before: #{schema_name} from path #{parsed_schema_names[schema_name]}, now at #{file_path}"
+                log "ERROR: We have encountered this schema before: \
+                  #{schema_name} from path \
+                  #{parsed_schema_names[schema_name]}, now at #{file_path}"
                 next
               else
                 parsed_schema_names[schema_name] = file_path
@@ -219,24 +239,24 @@ module Stepmod
               Dir["#{stepmod_path}/resources/#{schema_name}/descriptions.xml"].each do |description_xml_path|
                 log "INFO: Processing resources schema #{description_xml_path}"
                 description_document = Nokogiri::XML(File.read(description_xml_path)).root
-                description_document.xpath('//ext_description').each do |ext_description|
-
+                description_document.xpath("//ext_description").each do |ext_description|
                   # log "INFO: Processing linkend[#{ext_description['linkend']}]"
 
                   concept = Stepmod::Utils::Concept.parse(
                     ext_description,
                     reference_anchor: bibdata.anchor,
                     reference_clause: nil,
-                    file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                    file_path: Pathname.new(description_xml_path)
+                                .relative_path_from(stepmod_path),
                   )
                   next unless concept
 
-                  unless term_special_category(bibdata)
-                    # log "INFO: this part is generic"
-                    resource_concepts << concept
-                  else
+                  if term_special_category(bibdata)
                     # log "INFO: this part is special"
-                    current_part_resources << concept
+                    find_or_initialize_concept(current_part_resources, concept)
+                  else
+                    # log "INFO: this part is generic"
+                    find_or_initialize_concept(resource_concepts, concept)
                   end
 
                   parsed_bibliography << bibdata
@@ -248,9 +268,11 @@ module Stepmod
             log "INFO: Processing module.xml for #{file_path}"
             # Assumption: every schema is only linked by a single module document.
             # puts current_document.xpath('//module').length
-            schema_name = current_document.xpath('//module').first['name']
+            schema_name = current_document.xpath("//module").first["name"]
             if parsed_schema_names[schema_name]
-              log "ERROR: We have encountered this schema before: #{schema_name} from path #{parsed_schema_names[schema_name]}, now at #{file_path}"
+              log "ERROR: We have encountered this schema before: \
+                #{schema_name} from path #{parsed_schema_names[schema_name]}, \
+                  now at #{file_path}"
               next
             else
               parsed_schema_names[schema_name] = file_path
@@ -260,20 +282,25 @@ module Stepmod
             log "INFO: Processing modules schema #{description_xml_path}"
 
             if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(File.read(description_xml_path)).root
-              description_document.xpath('//ext_description').each do |ext_description|
-
-                linkend_schema = ext_description['linkend'].split('.').first
+              description_document = Nokogiri::XML(
+                File.read(description_xml_path),
+              )
+                .root
+              description_document.xpath("//ext_description").each do |ext_description|
+                linkend_schema = ext_description["linkend"].split(".").first
                 concept = Stepmod::Utils::Concept.parse(
                   ext_description,
                   reference_anchor: bibdata.anchor,
                   reference_clause: nil,
-                  file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                  file_path: Pathname.new(description_xml_path)
+                              .relative_path_from(stepmod_path),
                 )
                 next unless concept
 
-                current_part_modules_arm[linkend_schema] ||= []
-                current_part_modules_arm[linkend_schema] << concept
+                current_part_modules_arm[linkend_schema] ||= Glossarist::Collection.new
+                find_or_initialize_concept(
+                  current_part_modules_arm[linkend_schema], concept
+                )
                 # puts part_modules_arm.inspect
                 parsed_bibliography << bibdata
               end
@@ -283,21 +310,28 @@ module Stepmod
             log "INFO: Processing modules schema #{description_xml_path}"
 
             if File.exists?(description_xml_path)
-              description_document = Nokogiri::XML(File.read(description_xml_path)).root
-              description_document.xpath('//ext_description').each do |ext_description|
-
-                linkend_schema = ext_description['linkend'].split('.').first
+              description_document = Nokogiri::XML(
+                File.read(description_xml_path),
+              )
+                .root
+              description_document.xpath("//ext_description").each do |ext_description|
+                linkend_schema = ext_description["linkend"].split(".").first
 
                 concept = Stepmod::Utils::Concept.parse(
                   ext_description,
                   reference_anchor: bibdata.anchor,
                   reference_clause: nil,
-                  file_path: Pathname.new(description_xml_path).relative_path_from(stepmod_path)
+                  file_path: Pathname
+                              .new(description_xml_path)
+                              .relative_path_from(stepmod_path),
                 )
                 next unless concept
 
-                current_part_modules_mim[linkend_schema] ||= []
-                current_part_modules_mim[linkend_schema] << concept
+                current_part_modules_mim[linkend_schema] ||=
+                  Glossarist::Collection.new
+                find_or_initialize_concept(
+                  current_part_modules_mim[linkend_schema], concept
+                )
 
                 parsed_bibliography << bibdata
               end
@@ -306,21 +340,39 @@ module Stepmod
           end
 
           log "INFO: Completed processing XML file #{fpath}"
-          if current_part_concepts.empty?
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) because it contains no concepts."
-          elsif current_part_concepts.length < 3
-            log "INFO: Skipping #{fpath} (#{bibdata.docid}) because it only has #{current_part_concepts.length} terms."
-
-            current_part_concepts.each do |x|
-              general_concepts << x
+          if current_part_concepts.to_a.empty?
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
+              because it contains no concepts."
+          elsif current_part_concepts.to_a.length < 3
+            log "INFO: Skipping #{fpath} (#{bibdata.docid}) \
+              because it only has #{current_part_concepts.to_a.length} terms."
+
+            current_part_concepts.to_a.each do |x|
+              general_concepts.store(x)
             end
           else
-            part_concepts << [bibdata, current_part_concepts] unless current_part_concepts.empty?
+            unless current_part_concepts.to_a.empty?
+              part_concepts << [bibdata,
+                                current_part_concepts]
+            end
+          end
+          unless current_part_resources.to_a.empty?
+            part_resources << [bibdata,
+                               current_part_resources]
+          end
+          if (current_part_modules_arm.to_a.size +
+              current_part_modules_mim.to_a.size).positive?
+            part_modules << [bibdata, current_part_modules_arm,
+                             current_part_modules_mim]
           end
-          part_resources << [bibdata, current_part_resources] unless current_part_resources.empty?
-          part_modules << [bibdata, current_part_modules_arm, current_part_modules_mim] if current_part_modules_arm.size + current_part_modules_mim.size > 0
         end
       end
+
+      def find_or_initialize_concept(collection, localized_concept)
+        concept = collection
+          .store(Glossarist::Concept.new(id: SecureRandom.uuid))
+        concept.add_l10n(localized_concept)
+      end
     end
   end
 end