glossarist 2.6.6 → 2.6.7

data/lib/glossarist/cli/validate_command.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "paint"
+require "table_tennis"
+
 module Glossarist
   class CLI
     class ValidateCommand
@@ -9,47 +12,135 @@ module Glossarist
       end
 
       def run
-        result = DatasetValidator.new.validate(
+        text_output = @options[:format] == "text"
+        validator = DatasetValidator.new(on_progress: text_output ? method(:print_progress) : nil)
+        result = validator.validate(
           @path,
           strict: @options[:strict],
           reference_path: @options[:reference_path],
         )
+
+        $stderr.print "\r#{' ' * 60}\r" if text_output
         report(result)
-        exit_code = result.errors.any? || (@options[:strict] && result.warnings.any?) ? 1 : 0
-        exit(exit_code) unless exit_code.zero?
+        exit(1) unless result.errors.empty? && !strict_failure?(result)
       end
 
       private
 
+      def strict_failure?(result)
+        @options[:strict] && result.warnings.any?
+      end
+
+      def print_progress(current, total)
+        pct = (current.to_f / total * 100).round
+        bar_width = 30
+        filled = (current.to_f / total * bar_width).round
+        bar = "#{'█' * filled}#{'░' * (bar_width - filled)}"
+
+        $stderr.print "\r  #{Paint['Validating', :bold]} #{bar} #{current}/#{total} (#{pct}%)"
+        $stderr.flush
+      end
+
       def report(result)
         case @options[:format]
         when "json"
-          require "json"
-          puts JSON.pretty_generate(result.to_h)
+          puts result.to_json
         when "yaml"
-          require "yaml"
-          puts YAML.dump(result.to_h)
+          puts result.to_yaml
         else
-          report_text(result)
+          print_text_output(result)
+          print_table_output(result) if result.issues.any?
         end
       end
 
-      def report_text(result)
-        if result.valid?
-          puts "Valid."
-        else
-          puts "Invalid."
-          result.errors.each { |e| puts "  ERROR: #{e}" }
+      def print_text_output(result)
+        puts
+        puts Paint["Validating #{@path}", :bold]
+        puts
+
+        if result.issues.empty?
+          puts "  #{Paint['No issues found.', :green, :bold]}"
+          return
         end
 
-        if result.warnings.any?
-          result.warnings.each do |w|
-            puts "  WARNING: #{w}"
-          end
+        print_grouped_issues(result)
+        print_summary_line(result)
+      end
+
+      def print_grouped_issues(result)
+        result.issues
+          .group_by { |i| i.location || "(dataset)" }
+          .sort_by { |loc, issues| [has_errors?(issues) ? 0 : 1, loc] }
+          .each { |location, issues| print_location_group(location, issues) }
+      end
+
+      def has_errors?(issues)
+        issues.any?(&:error?)
+      end
+
+      def print_location_group(location, issues)
+        puts "  #{Paint[location, :cyan, :bold]}"
+        issues.sort_by { |i| issue_sort_key(i) }
+          .each { |issue| print_issue(issue) }
+        puts
+      end
+
+      def issue_sort_key(issue)
+        [issue.error? ? 0 : 1, issue.code || "￿", issue.message]
+      end
+
+      def print_issue(issue)
+        color = issue.error? ? :red : :yellow
+        label = Paint[issue.error? ? "ERROR" : " WARN", color, :bold]
+        code = Paint["%-8s" % (issue.code || ""), :magenta]
+        msg_col = 21
+
+        puts "    #{label}  #{code}  #{issue.message}"
+        puts "#{' ' * msg_col}#{Paint[issue.suggestion, :green]}" if issue.suggestion
+      end
+
+      def print_summary_line(result)
+        error_count = result.issues.count(&:error?)
+        warning_count = result.issues.count(&:warning?)
+
+        status = error_count.positive? ? Paint["INVALID", :red, :bold] : Paint["VALID", :green, :bold]
+
+        details = []
+        details << Paint["#{error_count} error(s)", :red] if error_count.positive?
+        details << Paint["#{warning_count} warning(s)", :yellow] if warning_count.positive?
+
+        puts "  #{status}  #{details.join(', ')}"
+      end
+
+      def print_table_output(result)
+        rows = build_summary_rows(result)
+        return if rows.empty?
+
+        options = {
+          title: "Issues by Rule",
+          columns: %i[code severity count],
+          headers: { code: "Rule", severity: "Level", count: "Count" },
+          color_scales: { count: :gw },
+          mark: ->(row) { row[:severity] == "error" },
+          zebra: true,
+        }
+        puts
+        puts TableTennis.new(rows, options)
+      end
+
+      def build_summary_rows(result)
+        counts = Hash.new(0)
+        severities = {}
+
+        result.issues.each do |issue|
+          key = issue.code || "unknown"
+          counts[key] += 1
+          severities[key] ||= issue.severity
         end
 
-        total = result.errors.length + result.warnings.length
-        puts "#{total} issue(s) found."
+        counts.sort_by { |_, c| -c }.map do |code, count|
+          { code: code, severity: severities[code], count: count }
+        end
       end
     end
   end

data/lib/glossarist/cli.rb

@@ -9,6 +9,7 @@ module Glossarist
     autoload :ValidateCommand, "#{__dir__}/cli/validate_command"
     autoload :ImportCommand,   "#{__dir__}/cli/import_command"
     autoload :ExportCommand,   "#{__dir__}/cli/export_command"
+    autoload :CompareCommand,  "#{__dir__}/cli/compare_command"
     desc "generate_latex", "Convert Concepts to Latex format"
 
     option :concepts_path, aliases: :p, required: true,
@@ -120,6 +121,23 @@ module Glossarist
       CLI::ExportCommand.new(path, options).run
     end
 
+    desc "compare NEW_PATH OLD_PATH", "Compare two concept datasets"
+    option :format, type: :string, default: "text",
+                    enum: %w[text json yaml],
+                    desc: "Output format"
+    option :report, type: :string,
+                    desc: "Write report to file"
+    option :no_diffs, type: :boolean, default: false,
+                      desc: "Skip per-concept diff computation"
+    def compare(new_path, old_path)
+      CLI::CompareCommand.new(new_path, old_path, options).run
+    end
+
+    desc "version", "Show Glossarist version"
+    def version
+      puts Glossarist::VERSION
+    end
+
     def method_missing(*args)
       warn "No method found named: #{args[0]}"
       warn "Run with `--help` or `-h` to see available options"

data/lib/glossarist/collections/bibliography_collection.rb

@@ -39,9 +39,11 @@ module Glossarist
       def populate_bibliographies(concepts)
         concepts.each do |concept|
           concept.default_lang.sources.each do |source|
-            next if source.origin.text.nil?
+            next if source.origin.ref.nil?
+            ref_text = source.origin.ref.source
+            next if ref_text.nil?
 
-            fetch(source.origin.text)
+            fetch(ref_text)
           end
         end
       end

data/lib/glossarist/collections/localization_collection.rb

@@ -7,6 +7,8 @@ module Glossarist
 
       index_by :language_code
 
+      alias :length :size
+
       def [](lang_code)
         find_by(:language_code, lang_code.to_s)
       end

data/lib/glossarist/comparison_result.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ComparisonResult < Lutaml::Model::Serializable
+    attribute :new_count, :integer
+    attribute :old_count, :integer
+    attribute :matched, :string, collection: true, initialize_empty: true
+    attribute :new_only, :string, collection: true, initialize_empty: true
+    attribute :old_only, :string, collection: true, initialize_empty: true
+    attribute :diffs, ConceptDiff, collection: true, initialize_empty: true
+
+    key_value do
+      map :new_count, to: :new_count
+      map :old_count, to: :old_count
+      map :matched, to: :matched
+      map :new_only, to: :new_only
+      map :old_only, to: :old_only
+      map :diffs, to: :diffs
+    end
+
+    def summary
+      diff = new_count - old_count
+      change = if diff.positive?
+                 "+#{diff} new"
+               elsif diff.negative?
+                 "#{diff.abs} removed"
+               else
+                 "no change"
+               end
+      "#{new_count} new, #{old_count} old (#{change}), " \
+        "#{matched.length} matched, #{new_only.length} new-only, " \
+        "#{old_only.length} old-only"
+    end
+  end
+end

data/lib/glossarist/concept_collector.rb

@@ -35,9 +35,37 @@ module Glossarist
       end
     end
 
+    def self.count(dir)
+      dir = File.expand_path(dir)
+      return 0 unless File.directory?(dir)
+
+      if managed_concepts?(dir)
+        Dir.glob(File.join(dir, "concepts", "concept", "*.yaml")).length
+      elsif v2_concepts?(dir)
+        count_v2(dir)
+      elsif v1_concepts?(dir)
+        Dir.glob(File.join(dir, "concepts", "*.yaml")).length
+      else
+        0
+      end
+    end
+
     class << self
       private
 
+      def count_v2(dir)
+        if v2_flat_concepts?(dir)
+          Dir.glob(File.join(dir, "concepts", "*.yaml")).length
+        else
+          v2_dir = File.join(dir, "geolexica-v2")
+          if File.directory?(File.join(v2_dir, "concepts"))
+            Dir.glob(File.join(v2_dir, "concepts", "concept", "*.yaml")).length
+          else
+            Dir.glob(File.join(v2_dir, "*.yaml")).length
+          end
+        end
+      end
+
       def v1_concepts?(dir)
         concepts_dir = File.join(dir, "concepts")
         File.directory?(concepts_dir) &&
@@ -124,6 +152,7 @@ module Glossarist
       def each_grouped_v2_concepts(v2_dir, &block)
         collection = ManagedConceptCollection.new
         manager = ConceptManager.new(path: v2_dir)
+        manager.version = detect_schema_version(v2_dir)
         manager.load_from_files(collection: collection)
         collection.each(&block)
       end
@@ -131,6 +160,7 @@ module Glossarist
       def collect_grouped_v2_concepts(v2_dir)
         collection = ManagedConceptCollection.new
         manager = ConceptManager.new(path: v2_dir)
+        manager.version = detect_schema_version(v2_dir)
         manager.load_from_files(collection: collection)
         collection.to_a
       end
@@ -178,6 +208,20 @@ module Glossarist
         end
         nil
       end
+
+      def detect_schema_version(dir)
+        concepts_dir = File.join(dir, "concepts")
+        search_dir = File.directory?(concepts_dir) ? concepts_dir : dir
+        sample = Dir.glob(File.join(search_dir, "*.yaml")).first
+        return "2" unless sample
+
+        raw = File.read(sample, encoding: "utf-8")
+        doc = ConceptDocument.from_yamls(raw)
+        ver = doc.concept&.schema_version.to_s
+        ver == "3" ? "3" : "2"
+      rescue StandardError
+        "2"
+      end
     end
   end
 end

data/lib/glossarist/concept_comparator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptComparator
+    def initialize(new_concepts:, old_concepts:)
+      @new_concepts = new_concepts
+      @old_concepts = old_concepts
+    end
+
+    def compare(show_diffs: true)
+      new_index = build_index(@new_concepts)
+      old_index = build_index(@old_concepts)
+
+      matched_ids = new_index.keys & old_index.keys
+      new_only_ids = new_index.keys - old_index.keys
+      old_only_ids = old_index.keys - new_index.keys
+
+      diffs = if show_diffs
+                compute_diffs(matched_ids, new_index, old_index)
+              else
+                []
+              end
+
+      ComparisonResult.new(
+        new_count: @new_concepts.length,
+        old_count: @old_concepts.length,
+        matched: matched_ids.sort,
+        new_only: new_only_ids.sort,
+        old_only: old_only_ids.sort,
+        diffs: diffs,
+      )
+    end
+
+    private
+
+    def build_index(concepts)
+      concepts.each_with_object({}) do |concept, index|
+        id = extract_id(concept)
+        index[id] = concept if id
+      end
+    end
+
+    def extract_id(concept)
+      concept.data&.id || concept.id
+    end
+
+    def compute_diffs(matched_ids, new_index, old_index)
+      matched_ids.filter_map do |id|
+        new_concept = new_index[id]
+        old_concept = old_index[id]
+
+        score, tree = Lutaml::Model::Serialize.diff_with_score(
+          new_concept, old_concept,
+          show_unchanged: false,
+          highlight_diff: false,
+          indent: ""
+        )
+        similarity = ((1 - score) * 100).round(2)
+
+        ConceptDiff.new(
+          concept_id: id,
+          similarity: similarity,
+          diff_tree: strip_ansi(tree),
+        )
+      end.sort_by { |d| -d.similarity }
+    end
+
+    def strip_ansi(text)
+      text.gsub(/\e\[[0-9;]*m/, "")
+    end
+  end
+end

data/lib/glossarist/concept_data.rb

@@ -78,5 +78,21 @@ module Glossarist
 
       sources.select(&:authoritative?)
     end
+
+    def all_sources
+      list = sources.to_a
+      definition.each { |d| list.concat(d.sources.to_a) }
+      notes.each { |n| list.concat(n.sources.to_a) }
+      examples.each { |e| list.concat(e.sources.to_a) }
+      list
+    end
+
+    def text_content
+      texts = []
+      definition.each { |d| texts << d.content if d.content }
+      notes.each { |n| texts << n.content if n.content }
+      examples.each { |e| texts << e.content if e.content }
+      texts
+    end
   end
 end

data/lib/glossarist/concept_diff.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptDiff < Lutaml::Model::Serializable
+    attribute :concept_id, :string
+    attribute :similarity, :float
+    attribute :diff_tree, :string
+
+    key_value do
+      map :concept_id, to: :concept_id
+      map :similarity, to: :similarity
+      map :diff_tree, to: :diff_tree
+    end
+  end
+end

data/lib/glossarist/concept_document.rb

@@ -20,6 +20,17 @@ module Glossarist
       )
     end
 
+    def self.for_version(version)
+      case version.to_s
+      when "2"
+        require_relative "v2"
+        V2::ConceptDocument
+      else
+        require_relative "v3"
+        V3::ConceptDocument
+      end
+    end
+
     def to_managed_concept
       mc = concept
       localizations.each { |l10n| mc.add_localization(l10n) }

data/lib/glossarist/concept_manager.rb

@@ -2,6 +2,7 @@ module Glossarist
   class ConceptManager < Lutaml::Model::Serializable
     attribute :path, :string
     attribute :localized_concepts_path, :string
+    attribute :version, :string, default: -> { "2" }
 
     key_value do
       map :path, to: :path
@@ -9,6 +10,19 @@ module Glossarist
           to: :localized_concepts_path
     end
 
+    def concept_document_class
+      ConceptDocument.for_version(version)
+    end
+
+    def localized_concept_class
+      if version.to_s == "2"
+        require_relative "v2"
+        V2::LocalizedConcept
+      else
+        LocalizedConcept
+      end
+    end
+
     def load_from_files(collection: nil)
       collection ||= ManagedConceptCollection.new
 
@@ -35,7 +49,7 @@ module Glossarist
 
     def load_concept_from_file(filename) # rubocop:disable Metrics/CyclomaticComplexity
       raw = File.read(filename, encoding: "utf-8")
-      doc = ConceptDocument.from_yamls(raw)
+      doc = concept_document_class.from_yamls(raw)
       concept = doc.concept
       unless concept
         raise Glossarist::ParseError.new(filename: filename)
@@ -44,7 +58,7 @@ module Glossarist
       concept_uuid = concept.identifier || concept.data&.id || File.basename(
         filename, ".*"
       )
-      concept.instance_variable_set(:@uuid, concept_uuid)
+      concept.uuid = concept_uuid
 
       concept.data.localized_concepts.each_value do |id|
         localized_concept = load_localized_concept(id, doc.localizations)
@@ -60,15 +74,15 @@ module Glossarist
       if inline_localizations
         l10n = inline_localizations.find { |l| l.id == id }
         if l10n
-          l10n.instance_variable_set(:@uuid, id)
+          l10n.uuid = id
           return l10n
         end
       end
 
-      l10n = LocalizedConcept.from_yaml(
+      l10n = localized_concept_class.from_yaml(
         File.read(localized_concept_path(id), encoding: "utf-8"),
       )
-      l10n.instance_variable_set(:@uuid, id)
+      l10n.uuid = id
       l10n
     rescue Psych::SyntaxError => e
       raise Glossarist::ParseError.new(filename: filename, line: e.line)

data/lib/glossarist/concept_ref.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptRef < Lutaml::Model::Serializable
+    attribute :source, :string
+    attribute :id, :string
+
+    key_value do
+      map :source, to: :source
+      map :id, to: :id
+    end
+  end
+end

data/lib/glossarist/concept_validator.rb

@@ -4,8 +4,9 @@ module Glossarist
   class ConceptValidator
     attr_reader :path, :errors, :warnings
 
-    def initialize(path)
+    def initialize(path, on_progress: nil)
       @path = path
+      @on_progress = on_progress
       @errors = []
       @warnings = []
     end
@@ -14,9 +15,12 @@ module Glossarist
       result = ValidationResult.new
       context = Validation::Rules::DatasetContext.new(@path)
       concept_rules = Validation::Rules::Registry.for_scope(:concept)
+      total = ConceptCollector.count(@path)
       file_idx = 0
 
       ConceptCollector.each_concept(@path) do |concept|
+        context.add_concept(concept)
+
         fname = concept_file_name(concept, file_idx)
         concept_context = Validation::Rules::ConceptContext.new(
           concept, file_name: fname, collection_context: context
@@ -29,6 +33,7 @@ module Glossarist
         end
 
         file_idx += 1
+        @on_progress&.call(file_idx, total)
       end
 
       if file_idx.zero?

data/lib/glossarist/context_configuration.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module ContextConfiguration
+    def context_id
+      self::CONTEXT_ID
+    end
+
+    def context
+      Lutaml::Model::GlobalContext.context(context_id)
+    end
+
+    def create_context(
+      id:,
+      registry: nil,
+      fallback_to: [context_id],
+      substitutions: []
+    )
+      normalized_id = id.to_sym
+
+      return populate_context! if normalized_id == context_id
+
+      Lutaml::Model::GlobalContext.unregister_context(normalized_id) if Lutaml::Model::GlobalContext.context(normalized_id)
+      create_type_context(
+        id: normalized_id,
+        registry: registry || Lutaml::Model::TypeRegistry.new,
+        fallback_to: normalize_fallbacks(fallback_to),
+        substitutions: substitutions,
+      )
+    end
+
+    def populate_context!
+      Lutaml::Model::GlobalContext.unregister_context(context_id) if context
+      register_models_in(base_type_context)
+    end
+
+    def register_model(klass, id:)
+      normalized_id = id.to_sym
+      registered_models[normalized_id] = klass
+      (context || populate_base_context).registry.register(normalized_id, klass)
+      klass
+    end
+
+    def resolve_model(id)
+      Lutaml::Model::GlobalContext.resolve_type(id, context_id)
+    end
+
+    private
+
+    def populate_base_context
+      base_type_context
+    end
+
+    def create_type_context(id:, registry:, fallback_to:, substitutions: [])
+      Lutaml::Model::GlobalContext.create_context(
+        id: id,
+        registry: registry,
+        fallback_to: fallback_to,
+        substitutions: substitutions,
+      ).tap do
+        Lutaml::Model::GlobalContext.clear_caches
+      end
+    end
+
+    def base_type_context
+      create_type_context(
+        id: context_id,
+        registry: Lutaml::Model::TypeRegistry.new,
+        fallback_to: [:default],
+      )
+    end
+
+    def register_models_in(type_context)
+      registered_models.each do |model_id, klass|
+        type_context.registry.register(model_id, klass)
+      end
+
+      Lutaml::Model::GlobalContext.clear_caches
+      type_context
+    end
+
+    def normalize_fallbacks(fallback_to)
+      Array(fallback_to).map(&:to_sym)
+    end
+
+    def registered_models
+      @registered_models ||= {}
+    end
+  end
+end