metanorma-plugin-glossarist 0.2.3 → 0.2.5

data/Rakefile

@@ -3,4 +3,4 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/lib/metanorma/plugin/glossarist/dataset_preprocessor.rb

@@ -6,15 +6,6 @@ require "liquid"
 require "asciidoctor/reader"
 require "glossarist"
 require "metanorma/plugin/glossarist/document"
-require "liquid/custom_blocks/with_glossarist_context"
-
-require "liquid/drops/concepts_drop"
-require "liquid/custom_filters/filters"
-
-Liquid::Template
-  .register_tag("with_glossarist_context",
-                Liquid::CustomBlocks::WithGlossaristContext)
-Liquid::Template.register_filter(Liquid::CustomFilters::Filters)
 
 module Metanorma
   module Plugin
@@ -23,12 +14,10 @@ module Metanorma
         GLOSSARIST_DATASET_REGEX = /^:glossarist-dataset:\s*(.*?)$/m.freeze
         GLOSSARIST_IMPORT_REGEX = /^glossarist::import\[(.*?)\]$/m.freeze
         GLOSSARIST_RENDER_REGEX = /^glossarist::render\[(.*?)\]$/m.freeze
-
-        GLOSSARIST_BLOCK_REGEX = /^\[glossarist,(.+?),(.+?)\]/
-        GLOSSARIST_FILTER_BLOCK_REGEX = /^\[glossarist,(.+?),(filter=.+?),(.+?)\]/
-
-        GLOSSARIST_BIBLIOGRAPHY_REGEX = /^glossarist::render_bibliography\[(.*?)\]$/m.freeze
-        GLOSSARIST_BIBLIOGRAPHY_ENTRY_REGEX = /^glossarist::render_bibliography_entry\[(.*?)\]$/m.freeze
+        GLOSSARIST_BLOCK_REGEX = /^\[glossarist,(.+?),(.+?)\]/.freeze
+        GLOSSARIST_FILTER_BLOCK_REGEX = /^\[glossarist,(.+?),(filter=.+?),(.+?)\]/.freeze # rubocop:disable Layout/LineLength
+        GLOSSARIST_BIBLIOGRAPHY_REGEX = /^glossarist::render_bibliography\[(.*?)\]$/m.freeze # rubocop:disable Layout/LineLength
+        GLOSSARIST_BIBLIOGRAPHY_ENTRY_REGEX = /^glossarist::render_bibliography_entry\[(.*?)\]$/m.freeze # rubocop:disable Layout/LineLength
 
         # Search document for the following blocks
         #   - :glossarist-dataset: dataset1:./dataset1;dataset2:./dataset2
@@ -63,6 +52,7 @@ module Metanorma
           @title_depth = { value: 2 }
           @rendered_bibliographies = {}
           @seen_glossarist = []
+          @context_names = []
         end
 
         def process(document, reader)
@@ -83,14 +73,18 @@ module Metanorma
         protected
 
         def log(doc, text)
-          File.open("#{doc.attr('docfile')}.glossarist.log.txt", "w:UTF-8") do |f|
+          File.open("#{doc.attr('docfile')}.glossarist.log.txt",
+                    "w:UTF-8") do |f|
             f.write(text)
           end
         end
 
         private
 
-        def prepare_document(document, input_lines, end_mark = nil)
+        def prepare_document( # rubocop:disable Metrics/MethodLength
+          document, input_lines, end_mark = nil,
+          skip_dataset: false
+        )
           liquid_doc = Document.new
           liquid_doc.file_system = @config[:file_system]
 
@@ -98,13 +92,19 @@ module Metanorma
             current_line = input_lines.next
             break if end_mark && current_line == end_mark
 
-            process_line(document, input_lines, current_line, liquid_doc)
+            if !(
+              skip_dataset && current_line.start_with?(":glossarist-dataset:")
+            )
+              process_line(document, input_lines, current_line, liquid_doc)
+            end
           end
 
           liquid_doc
         end
 
-        def process_line(document, input_lines, current_line, liquid_doc)
+        def process_line( # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+          document, input_lines, current_line, liquid_doc
+        )
           if match = current_line.match(GLOSSARIST_DATASET_REGEX)
             process_dataset_tag(document, input_lines, liquid_doc, match)
           elsif match = current_line.match(GLOSSARIST_RENDER_REGEX)
@@ -117,40 +117,44 @@ module Metanorma
             process_bibliography_entry(liquid_doc, match)
           elsif match = current_line.match(GLOSSARIST_FILTER_BLOCK_REGEX)
             process_glossarist_block(document, liquid_doc, input_lines, match)
-          elsif match = current_line.match(GLOSSARIST_BLOCK_REGEX)
+          elsif match = current_line.match(GLOSSARIST_BLOCK_REGEX) # rubocop:disable Lint/DuplicateBranch
             process_glossarist_block(document, liquid_doc, input_lines, match)
           else
-            @title_depth[:value] = current_line.sub(/ .*$/, "").size if /^==+ \S/.match?(current_line)
+            if /^==+ \S/.match?(current_line)
+              @title_depth[:value] =
+                current_line.sub(/ .*$/,
+                                 "").size
+            end
             liquid_doc.add_content(current_line)
           end
         end
 
         def process_dataset_tag(document, input_lines, liquid_doc, match)
           @seen_glossarist << "x"
-          context_names = prepare_dataset_contexts(document, match[1])
-
+          @context_names << prepare_dataset_contexts(document, match[1])
+          @context_names.flatten!
           dataset_section = <<~TEMPLATE
-            {% with_glossarist_context #{context_names} %}
             #{prepare_document(document, input_lines)}
-            {% endwith_glossarist_context %}
           TEMPLATE
 
           liquid_doc.add_content(
             dataset_section,
-            render: true,
+            render: false,
           )
         end
 
         def process_glossarist_block(document, liquid_doc, input_lines, match)
           @seen_glossarist << "x"
           end_mark = input_lines.next
-          path = relative_file_path(document, match[1])
-
+          # path = relative_file_path(document, match[1])
           glossarist_params = prepare_glossarist_block_params(document, match)
 
           glossarist_section = <<~TEMPLATE.strip
             {% with_glossarist_context #{glossarist_params} %}
-            #{prepare_document(document, input_lines, end_mark)}
+            #{prepare_document(
+              document, input_lines, end_mark,
+              skip_dataset: true
+            )}
             {% endwith_glossarist_context %}
           TEMPLATE
 
@@ -161,7 +165,7 @@ module Metanorma
         end
 
         def prepare_glossarist_block_params(document, match)
-          path = relative_file_path(document, match[1])
+          path = get_context_path(document, match[1])
 
           if match[2].strip.start_with?("filter")
             filters = match[2].split("=")[1..-1].join("=").strip
@@ -173,6 +177,22 @@ module Metanorma
           end
         end
 
+        def get_context_path(document, key) # rubocop:disable Metrics/MethodLength
+          context_path = nil
+          # try to get context_path from glossarist-dataset definition
+          if @context_names && !@context_names.empty?
+            context_names = @context_names.map(&:strip)
+            context_path = context_names.find do |context|
+              context_name, = context.split("=")
+              context_name == key
+            end
+          end
+
+          return context_path.split("=").last.strip if context_path
+
+          relative_file_path(document, key)
+        end
+
         def process_render_tag(liquid_doc, match)
           @seen_glossarist << "x"
           context_name, concept_name = match[1].split(",")
@@ -182,13 +202,15 @@ module Metanorma
           )
         end
 
-        def process_import_tag(liquid_doc, match)
+        def process_import_tag(liquid_doc, match) # rubocop:disable Metrics/AbcSize
           @seen_glossarist << "x"
           context_name = match[1].strip
-          dataset = @datasets[context_name.strip]
+          dataset = @datasets[context_name]
 
           liquid_doc.add_content(
-            dataset.concepts.map do |concept_name, _concept|
+            dataset.map do |concept|
+              concept_name = concept.data.localizations["eng"].data
+                .terms[0].designation
               concept_template(context_name, concept_name)
             end.join("\n"),
           )
@@ -200,7 +222,7 @@ module Metanorma
           dataset = @datasets[dataset_name]
 
           liquid_doc.add_content(
-            dataset.concepts.map do |concept|
+            dataset.map do |concept|
               concept_bibliography(concept)
             end.compact.sort.join("\n"),
           )
@@ -209,17 +231,19 @@ module Metanorma
         def process_bibliography_entry(liquid_doc, match)
           @seen_glossarist << "x"
           dataset_name, concept_name = match[1].split(",").map(&:strip)
-          concept = @datasets[dataset_name][concept_name]
-
-          liquid_doc.add_content(
-            concept_bibliography(concept),
-          )
+          concept = get_concept(dataset_name, concept_name)
+          bibliography = concept_bibliography(concept)
+          if bibliography
+            liquid_doc.add_content(bibliography)
+          end
         end
 
-        def concept_bibliography(concept)
-          bibliography = concept["eng"]["sources"].map do |source|
-            ref = source["origin"]["ref"]
+        def concept_bibliography(concept) # rubocop:disable Metrics/AbcSize
+          sources = concept.data.localizations["eng"].data.sources
+          return nil if sources.nil? || sources.empty?
 
+          bibliography = sources.map do |source|
+            ref = source.origin.text
             next if @rendered_bibliographies[ref] || ref.nil? || ref.empty?
 
             @rendered_bibliographies[ref] = ref.gsub(/[ \/:]/, "_")
@@ -230,18 +254,16 @@ module Metanorma
         end
 
         def prepare_dataset_contexts(document, contexts)
-          context_names = contexts.split(";").map do |context|
+          contexts.split(";").map do |context|
             context_name, file_path = context.split(":").map(&:strip)
             path = relative_file_path(document, file_path)
 
             dataset = ::Glossarist::ManagedConceptCollection.new
             dataset.load_from_files(path)
-            @datasets[context_name] = Liquid::Drops::ConceptsDrop.new(dataset)
+            @datasets[context_name] = dataset.map(&:to_liquid)
 
             "#{context_name}=#{path}"
           end
-
-          context_names.join(",")
         end
 
         def relative_file_path(document, file_path)
@@ -255,10 +277,10 @@ module Metanorma
 
         def concept_template(dataset_name, concept_name)
           <<~CONCEPT_TEMPLATE
-            #{"=" * (@title_depth[:value] + 1)} {{ #{dataset_name}['#{concept_name}'].term }}
+            #{'=' * (@title_depth[:value] + 1)} #{concept_title(dataset_name, concept_name)}
             #{alt_terms(dataset_name, concept_name)}
 
-            {{ #{dataset_name}['#{concept_name}']['eng'].definition[0].content }}
+            #{concept_definition(dataset_name, concept_name)}
 
             #{examples(dataset_name, concept_name)}
 
@@ -268,51 +290,99 @@ module Metanorma
           CONCEPT_TEMPLATE
         end
 
-        def alt_terms(dataset_name, concept_name)
-          concept = @datasets[dataset_name][concept_name]
+        def get_concept(dataset_name, concept_name)
+          @datasets[dataset_name].find do |c|
+            c.data.localizations["eng"]
+              .data.terms[0]
+              .designation == concept_name
+          end
+        end
+
+        def concept_title(dataset_name, concept_name)
+          concept = get_concept(dataset_name, concept_name)
+          concept.data.localizations["eng"].data.terms[0].designation
+        end
+
+        def concept_definition(dataset_name, concept_name)
+          concept = get_concept(dataset_name, concept_name)
+          definition = concept.data.localizations["eng"].data
+            .definition[0].content
+          definition.to_s
+        end
+
+        def alt_terms(dataset_name, concept_name) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          concept = get_concept(dataset_name, concept_name)
           term_types = %w[preferred admitted deprecated]
+          concept_terms = concept.data.localizations["eng"].data.terms
 
-          concept["eng"]["terms"][1..-1].map do |term|
-            type = if term_types.include?(term["normative_status"])
-                     term["normative_status"]
+          concept_terms[1..-1].map do |term|
+            type = if term_types.include?(term.normative_status)
+                     term.normative_status
                    else
                      "alt"
                    end
-            "#{type}:[#{term['designation']}]"
+            "#{type}:[#{term.designation}]"
           end.join("\n")
         end
 
-        def examples(dataset_name, concept_name)
-          <<~EXAMPLES
-            {% for example in #{dataset_name}['#{concept_name}']['eng'].examples %}
-            [example]
-            {{ example.content }}
+        def examples(dataset_name, concept_name) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          concept = get_concept(dataset_name, concept_name)
+          examples = concept.data.localizations["eng"].data.examples
+          result = []
+
+          examples.each do |example|
+            content = <<~EXAMPLE
+              [example]
+              #{example.content}
+
+            EXAMPLE
+            result << content
+          end
 
-            {% endfor %}
-          EXAMPLES
+          result.join("\n")
         end
 
-        def notes(dataset_name, concept_name)
-          <<~NOTES
-            {% for note in #{dataset_name}['#{concept_name}']['eng'].notes %}
-            [NOTE]
-            ====
-            {{ note.content }}
-            ====
+        def notes(dataset_name, concept_name) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          concept = get_concept(dataset_name, concept_name)
+          notes = concept.data.localizations["eng"].data.notes
+          result = []
 
-            {% endfor %}
-          NOTES
+          notes.each do |note|
+            content = <<~NOTE
+              [NOTE]
+              ====
+              #{note.content}
+              ====
+
+            NOTE
+            result << content
+          end
+
+          result.join("\n")
         end
 
-        def sources(dataset_name, concept_name)
-          <<~SOURCES
-            {% for source in #{dataset_name}['#{concept_name}']['eng'].sources %}
-            {%- if source.origin.ref == nil or source.origin.ref == '' %}{% continue %}{% endif %}
-            [.source]
-            <<{{ source.origin.ref | replace: ' ', '_' | replace: '/', '_' | replace: ':', '_' }},{{ source.origin.clause }}>>
+        def sources(dataset_name, concept_name) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          concept = get_concept(dataset_name, concept_name)
+          sources = concept.data.localizations["eng"].data.sources
+          result = []
+
+          sources.each do |source|
+            if source.origin.text && source.origin.text != ""
+              source_origin_text = source.origin.text
+                .gsub(" ", "_")
+                .gsub("/", "_")
+                .gsub(":", "_")
+              source_content = "#{source_origin_text},#{source.origin.clause}"
+              content = <<~SOURCES
+                [.source]
+                <<#{source_content}>>
+
+              SOURCES
+              result << content
+            end
+          end
 
-            {% endfor %}
-          SOURCES
+          result.join("\n")
         end
       end
     end

data/lib/metanorma/plugin/glossarist/document.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "liquid/custom_blocks/with_glossarist_context"
+require_relative "liquid/custom_filters/filters"
+
 module Metanorma
   module Plugin
     module Glossarist
@@ -24,14 +27,30 @@ module Metanorma
         end
 
         def render_liquid(file_content)
-          template = Liquid::Template.parse(file_content)
+          template = ::Liquid::Template
+            .parse(file_content, environment: create_liquid_environment)
+
           template.registers[:file_system] = file_system
-          rendered_template = template.render(strict_variables: false, error_mode: :warn)
+          rendered_template = template.render(strict_variables: false,
+                                              error_mode: :warn)
 
           return rendered_template unless template.errors.any?
 
           raise template.errors.first.cause
         end
+
+        def create_liquid_environment
+          ::Liquid::Environment.new.tap do |liquid_env|
+            liquid_env.register_tag(
+              "with_glossarist_context",
+              ::Metanorma::Plugin::Glossarist::Liquid::CustomBlocks::
+              WithGlossaristContext,
+            )
+            liquid_env.register_filter(
+              ::Metanorma::Plugin::Glossarist::Liquid::CustomFilters::Filters,
+            )
+          end
+        end
       end
     end
   end

data/lib/metanorma/plugin/glossarist/liquid/custom_blocks/with_glossarist_context.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "liquid"
+
+module Metanorma
+  module Plugin
+    module Glossarist
+      module Liquid
+        module CustomBlocks
+          class WithGlossaristContext < ::Liquid::Block
+            def initialize(tag_name, markup, tokens) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+              super
+
+              @contexts = []
+              @filters = {}
+
+              contexts, filters = markup.split(";", 2)
+
+              if filters && !filters.empty?
+                filters.strip.gsub(/^['"]|['"]$/,
+                                   "").split(";").each do |filter|
+                  property, value = filter.split("=")
+
+                  @filters[property] = value
+                end
+              end
+
+              contexts.split(",").each do |context|
+                context_name, file_path = context.split("=").map(&:strip)
+
+                @contexts << {
+                  name: context_name,
+                  file_path: file_path,
+                }
+              end
+            end
+
+            def load_collection(folder_path)
+              @@collections ||= {}
+
+              return @@collections[folder_path] if @@collections[folder_path]
+
+              collection = ::Glossarist::ManagedConceptCollection.new
+              collection.load_from_files(folder_path)
+              @@collections[folder_path] = collection
+            end
+
+            def filter_collection(collection, filters)
+              return collection unless filters
+
+              concept_filters = filters.dup
+              lang_filter = concept_filters.delete("lang")
+              group_filter = concept_filters.delete("group")
+              sort_filter = concept_filters.delete("sort_by")
+
+              collection = apply_lang_filter(collection, lang_filter)
+              collection = apply_group_filter(collection, group_filter)
+              collection = apply_field_filter(collection, concept_filters)
+              apply_sort_filter(collection, sort_filter)
+            end
+
+            def apply_field_filter(collection, field_filter) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+              return collection if field_filter.nil? || field_filter.empty?
+
+              collection.select do |obj| # rubocop:disable Metrics/BlockLength
+                fields = field_filter.keys.first.split(".")
+                value = field_filter.values.first
+                start_with = false
+
+                # check if the last field is a start_with condition
+                if fields.last.start_with?("start_with(") &&
+                    fields.last.include?(")")
+                  start_with = true
+                  # Extract content between first '(' and last ')'
+                  f = fields.last
+                  value = f[(f.index("(") + 1)...f.rindex(")")]
+                  fields = fields[0..-2]
+                end
+
+                fields.each do |field|
+                  # contain index (i.e. field['abc'] or field[1])
+                  if field.include?("[")
+                    field, index = field[0..-2].split("[")
+
+                    index = if index.include?("'") || index.include?("\"")
+                              index.gsub(/['"]/, "")
+                            else
+                              index.to_i
+                            end
+
+                    obj = obj.send(field.to_sym)[index]
+                  else
+                    obj = obj.send(field.to_sym)
+                  end
+                end
+
+                # check if the object matches the value
+                if start_with
+                  obj.start_with?(value)
+                else
+                  obj == value
+                end
+              end
+            end
+
+            def apply_lang_filter(collection, lang_filter)
+              return collection unless lang_filter
+
+              collection.select do |concept|
+                concept.data.localizations.key?(lang_filter)
+              end
+            end
+
+            def apply_group_filter(collection, group_filter)
+              return collection unless group_filter
+
+              collection.select do |concept|
+                concept.data.groups.include?(group_filter)
+              end
+            end
+
+            def apply_sort_filter(collection, sort_filter)
+              return collection unless sort_filter
+
+              sort_filter = case sort_filter
+                            when "term"
+                              "default_designation"
+                            else
+                              sort_filter
+                            end
+
+              collection.sort_by do |concept|
+                concept.send(sort_filter.to_sym)
+              end
+            end
+
+            def render(context)
+              @contexts.each do |local_context|
+                context_file = local_context[:file_path].strip
+                collection = load_collection(context_file)
+
+                filtered_collection = filter_collection(collection, @filters)
+
+                context[local_context[:name]] = filtered_collection
+                  .map(&:to_liquid)
+              end
+
+              super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/glossarist/liquid/custom_filters/filters.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Plugin
+    module Glossarist
+      module Liquid
+        module CustomFilters
+          module Filters
+            def values(list)
+              list.values
+            end
+
+            def terminological_data(term)
+              result = []
+
+              result << "&lt;#{term['usage_info']}&gt;" if term["usage_info"]
+              result << extract_grammar_info(term)
+              result << term["geographical_area"]&.upcase
+
+              result.unshift(",") if result.compact.size.positive?
+
+              result.compact.join(" ")
+            end
+
+            def extract_grammar_info(term)
+              return unless term["grammar_info"]
+
+              grammar_info = []
+
+              term["grammar_info"].each do |info|
+                grammar_info << info["gender"]&.join(", ")
+                grammar_info << info["number"]&.join(", ")
+                grammar_info << extract_parts_of_speech(info)
+              end
+
+              grammar_info.join(" ")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/glossarist/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Glossarist
-      VERSION = "0.2.3".freeze
+      VERSION = "0.2.5".freeze
     end
   end
 end

data/metanorma-plugin-glossarist.gemspec

@@ -13,7 +13,8 @@ Gem::Specification.new do |spec|
   spec.license       = "BSD-2-Clause"
 
   # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added
+  # into git.
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject do |f|
       f.match(%r{^(test|spec|features)/})
@@ -23,7 +24,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = ">= 2.7.0" # rubocop:disable Gemspec/RequiredRubyVersion
+
   spec.add_dependency "asciidoctor", "~> 2.0.0"
-  spec.add_dependency "glossarist", "~> 2.0"
+  spec.add_dependency "glossarist", "~> 2.3.6"
   spec.add_dependency "liquid", "~> 5"
 end