glossarist 2.6.5 → 2.6.7

data/lib/glossarist/comparison_result.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ComparisonResult < Lutaml::Model::Serializable
+    attribute :new_count, :integer
+    attribute :old_count, :integer
+    attribute :matched, :string, collection: true, initialize_empty: true
+    attribute :new_only, :string, collection: true, initialize_empty: true
+    attribute :old_only, :string, collection: true, initialize_empty: true
+    attribute :diffs, ConceptDiff, collection: true, initialize_empty: true
+
+    key_value do
+      map :new_count, to: :new_count
+      map :old_count, to: :old_count
+      map :matched, to: :matched
+      map :new_only, to: :new_only
+      map :old_only, to: :old_only
+      map :diffs, to: :diffs
+    end
+
+    def summary
+      diff = new_count - old_count
+      change = if diff.positive?
+                 "+#{diff} new"
+               elsif diff.negative?
+                 "#{diff.abs} removed"
+               else
+                 "no change"
+               end
+      "#{new_count} new, #{old_count} old (#{change}), " \
+        "#{matched.length} matched, #{new_only.length} new-only, " \
+        "#{old_only.length} old-only"
+    end
+  end
+end

data/lib/glossarist/concept.rb

@@ -4,7 +4,7 @@ module Glossarist
     attribute :id, :string
     attribute :uuid, :string
     attribute :subject, :string
-    attribute :non_verb_rep, :string
+    attribute :non_verb_rep, NonVerbRep, collection: true
     attribute :extension_attributes, :string
     attribute :lineage_source, :string
     attribute :localizations, :hash

data/lib/glossarist/concept_collector.rb

@@ -35,9 +35,37 @@ module Glossarist
       end
     end
 
+    def self.count(dir)
+      dir = File.expand_path(dir)
+      return 0 unless File.directory?(dir)
+
+      if managed_concepts?(dir)
+        Dir.glob(File.join(dir, "concepts", "concept", "*.yaml")).length
+      elsif v2_concepts?(dir)
+        count_v2(dir)
+      elsif v1_concepts?(dir)
+        Dir.glob(File.join(dir, "concepts", "*.yaml")).length
+      else
+        0
+      end
+    end
+
     class << self
       private
 
+      def count_v2(dir)
+        if v2_flat_concepts?(dir)
+          Dir.glob(File.join(dir, "concepts", "*.yaml")).length
+        else
+          v2_dir = File.join(dir, "geolexica-v2")
+          if File.directory?(File.join(v2_dir, "concepts"))
+            Dir.glob(File.join(v2_dir, "concepts", "concept", "*.yaml")).length
+          else
+            Dir.glob(File.join(v2_dir, "*.yaml")).length
+          end
+        end
+      end
+
       def v1_concepts?(dir)
         concepts_dir = File.join(dir, "concepts")
         File.directory?(concepts_dir) &&
@@ -124,6 +152,7 @@ module Glossarist
       def each_grouped_v2_concepts(v2_dir, &block)
         collection = ManagedConceptCollection.new
         manager = ConceptManager.new(path: v2_dir)
+        manager.version = detect_schema_version(v2_dir)
         manager.load_from_files(collection: collection)
         collection.each(&block)
       end
@@ -131,6 +160,7 @@ module Glossarist
       def collect_grouped_v2_concepts(v2_dir)
         collection = ManagedConceptCollection.new
         manager = ConceptManager.new(path: v2_dir)
+        manager.version = detect_schema_version(v2_dir)
         manager.load_from_files(collection: collection)
         collection.to_a
       end
@@ -178,6 +208,20 @@ module Glossarist
         end
         nil
       end
+
+      def detect_schema_version(dir)
+        concepts_dir = File.join(dir, "concepts")
+        search_dir = File.directory?(concepts_dir) ? concepts_dir : dir
+        sample = Dir.glob(File.join(search_dir, "*.yaml")).first
+        return "2" unless sample
+
+        raw = File.read(sample, encoding: "utf-8")
+        doc = ConceptDocument.from_yamls(raw)
+        ver = doc.concept&.schema_version.to_s
+        ver == "3" ? "3" : "2"
+      rescue StandardError
+        "2"
+      end
     end
   end
 end

data/lib/glossarist/concept_comparator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptComparator
+    def initialize(new_concepts:, old_concepts:)
+      @new_concepts = new_concepts
+      @old_concepts = old_concepts
+    end
+
+    def compare(show_diffs: true)
+      new_index = build_index(@new_concepts)
+      old_index = build_index(@old_concepts)
+
+      matched_ids = new_index.keys & old_index.keys
+      new_only_ids = new_index.keys - old_index.keys
+      old_only_ids = old_index.keys - new_index.keys
+
+      diffs = if show_diffs
+                compute_diffs(matched_ids, new_index, old_index)
+              else
+                []
+              end
+
+      ComparisonResult.new(
+        new_count: @new_concepts.length,
+        old_count: @old_concepts.length,
+        matched: matched_ids.sort,
+        new_only: new_only_ids.sort,
+        old_only: old_only_ids.sort,
+        diffs: diffs,
+      )
+    end
+
+    private
+
+    def build_index(concepts)
+      concepts.each_with_object({}) do |concept, index|
+        id = extract_id(concept)
+        index[id] = concept if id
+      end
+    end
+
+    def extract_id(concept)
+      concept.data&.id || concept.id
+    end
+
+    def compute_diffs(matched_ids, new_index, old_index)
+      matched_ids.filter_map do |id|
+        new_concept = new_index[id]
+        old_concept = old_index[id]
+
+        score, tree = Lutaml::Model::Serialize.diff_with_score(
+          new_concept, old_concept,
+          show_unchanged: false,
+          highlight_diff: false,
+          indent: ""
+        )
+        similarity = ((1 - score) * 100).round(2)
+
+        ConceptDiff.new(
+          concept_id: id,
+          similarity: similarity,
+          diff_tree: strip_ansi(tree),
+        )
+      end.sort_by { |d| -d.similarity }
+    end
+
+    def strip_ansi(text)
+      text.gsub(/\e\[[0-9;]*m/, "")
+    end
+  end
+end

data/lib/glossarist/concept_data.rb

@@ -30,6 +30,8 @@ module Glossarist
     # Language code should be exactly 3 char long.
     # TODO: use min_length, max_length once added in lutaml-model
     attribute :language_code, :string, pattern: /^.{3}$/
+    attribute :script, :string
+    attribute :system, :string
     attribute :entry_status, :string
 
     key_value do
@@ -48,6 +50,8 @@ module Glossarist
       map :references, to: :references
       map :domain, to: :domain
       map %i[language_code languageCode], to: :language_code
+      map :script, to: :script
+      map :system, to: :system
       map %i[entry_status entryStatus], to: :entry_status
       map %i[review_date reviewDate], to: :review_date
       map %i[review_decision_date reviewDecisionDate], to: :review_decision_date
@@ -74,5 +78,21 @@ module Glossarist
 
       sources.select(&:authoritative?)
     end
+
+    def all_sources
+      list = sources.to_a
+      definition.each { |d| list.concat(d.sources.to_a) }
+      notes.each { |n| list.concat(n.sources.to_a) }
+      examples.each { |e| list.concat(e.sources.to_a) }
+      list
+    end
+
+    def text_content
+      texts = []
+      definition.each { |d| texts << d.content if d.content }
+      notes.each { |n| texts << n.content if n.content }
+      examples.each { |e| texts << e.content if e.content }
+      texts
+    end
   end
 end

data/lib/glossarist/concept_diff.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptDiff < Lutaml::Model::Serializable
+    attribute :concept_id, :string
+    attribute :similarity, :float
+    attribute :diff_tree, :string
+
+    key_value do
+      map :concept_id, to: :concept_id
+      map :similarity, to: :similarity
+      map :diff_tree, to: :diff_tree
+    end
+  end
+end

data/lib/glossarist/concept_document.rb

@@ -20,6 +20,17 @@ module Glossarist
       )
     end
 
+    def self.for_version(version)
+      case version.to_s
+      when "2"
+        require_relative "v2"
+        V2::ConceptDocument
+      else
+        require_relative "v3"
+        V3::ConceptDocument
+      end
+    end
+
     def to_managed_concept
       mc = concept
       localizations.each { |l10n| mc.add_localization(l10n) }

data/lib/glossarist/concept_manager.rb

@@ -2,6 +2,7 @@ module Glossarist
   class ConceptManager < Lutaml::Model::Serializable
     attribute :path, :string
     attribute :localized_concepts_path, :string
+    attribute :version, :string, default: -> { "2" }
 
     key_value do
       map :path, to: :path
@@ -9,6 +10,19 @@ module Glossarist
           to: :localized_concepts_path
     end
 
+    def concept_document_class
+      ConceptDocument.for_version(version)
+    end
+
+    def localized_concept_class
+      if version.to_s == "2"
+        require_relative "v2"
+        V2::LocalizedConcept
+      else
+        LocalizedConcept
+      end
+    end
+
     def load_from_files(collection: nil)
       collection ||= ManagedConceptCollection.new
 
@@ -35,7 +49,7 @@ module Glossarist
 
     def load_concept_from_file(filename) # rubocop:disable Metrics/CyclomaticComplexity
       raw = File.read(filename, encoding: "utf-8")
-      doc = ConceptDocument.from_yamls(raw)
+      doc = concept_document_class.from_yamls(raw)
       concept = doc.concept
       unless concept
         raise Glossarist::ParseError.new(filename: filename)
@@ -44,7 +58,7 @@ module Glossarist
       concept_uuid = concept.identifier || concept.data&.id || File.basename(
         filename, ".*"
       )
-      concept.instance_variable_set(:@uuid, concept_uuid)
+      concept.uuid = concept_uuid
 
       concept.data.localized_concepts.each_value do |id|
         localized_concept = load_localized_concept(id, doc.localizations)
@@ -60,15 +74,15 @@ module Glossarist
       if inline_localizations
         l10n = inline_localizations.find { |l| l.id == id }
         if l10n
-          l10n.instance_variable_set(:@uuid, id)
+          l10n.uuid = id
           return l10n
         end
       end
 
-      l10n = LocalizedConcept.from_yaml(
+      l10n = localized_concept_class.from_yaml(
         File.read(localized_concept_path(id), encoding: "utf-8"),
       )
-      l10n.instance_variable_set(:@uuid, id)
+      l10n.uuid = id
       l10n
     rescue Psych::SyntaxError => e
       raise Glossarist::ParseError.new(filename: filename, line: e.line)

data/lib/glossarist/concept_ref.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptRef < Lutaml::Model::Serializable
+    attribute :source, :string
+    attribute :id, :string
+
+    key_value do
+      map :source, to: :source
+      map :id, to: :id
+    end
+  end
+end

data/lib/glossarist/concept_reference.rb

@@ -1,17 +1,29 @@
 # frozen_string_literal: true
 
 module Glossarist
+  # A typed reference to another concept, either local (within the same
+  # glossary) or external (in another concept registry).
+  #
+  # Local references use +concept_id+ without +source+. External references
+  # use +source+ (a registry URN prefix) and +concept_id+ to identify the
+  # target concept, or a direct +urn+ field for formal URN references.
   class ConceptReference < Lutaml::Model::Serializable
     attribute :term, :string
     attribute :concept_id, :string
     attribute :source, :string
     attribute :ref_type, :string
+    attribute :urn, :string
 
     key_value do
       map :term, to: :term
       map :concept_id, to: :concept_id
       map :source, to: :source
       map :ref_type, to: :ref_type
+      map :urn, to: :urn
+    end
+
+    def self.domain(concept_id)
+      new(concept_id: concept_id, ref_type: "domain")
     end
 
     def local?
@@ -23,25 +35,6 @@ module Glossarist
       !local?
     end
 
-    def to_urn
-      return nil unless external?
-      return nil unless source && concept_id
-
-      case source
-      when /\Aurn:iec/ then "#{source}-#{concept_id}"
-      when /\Aurn:iso/ then "#{source}:term:#{concept_id}"
-      else "#{source}/#{concept_id}"
-      end
-    end
-
-    def to_gcr_hash
-      h = { "term" => term }
-      h["concept_id"] = concept_id if concept_id
-      h["source"] = source if source
-      h["ref_type"] = ref_type if ref_type
-      h.compact
-    end
-
     def dedup_key
       concept_id ? [source, concept_id] : [source, concept_id, term]
     end

data/lib/glossarist/concept_validator.rb

@@ -4,8 +4,9 @@ module Glossarist
   class ConceptValidator
     attr_reader :path, :errors, :warnings
 
-    def initialize(path)
+    def initialize(path, on_progress: nil)
       @path = path
+      @on_progress = on_progress
       @errors = []
       @warnings = []
     end
@@ -14,9 +15,12 @@ module Glossarist
       result = ValidationResult.new
       context = Validation::Rules::DatasetContext.new(@path)
       concept_rules = Validation::Rules::Registry.for_scope(:concept)
+      total = ConceptCollector.count(@path)
       file_idx = 0
 
       ConceptCollector.each_concept(@path) do |concept|
+        context.add_concept(concept)
+
         fname = concept_file_name(concept, file_idx)
         concept_context = Validation::Rules::ConceptContext.new(
           concept, file_name: fname, collection_context: context
@@ -29,6 +33,7 @@ module Glossarist
         end
 
         file_idx += 1
+        @on_progress&.call(file_idx, total)
       end
 
       if file_idx.zero?

data/lib/glossarist/context_configuration.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module ContextConfiguration
+    def context_id
+      self::CONTEXT_ID
+    end
+
+    def context
+      Lutaml::Model::GlobalContext.context(context_id)
+    end
+
+    def create_context(
+      id:,
+      registry: nil,
+      fallback_to: [context_id],
+      substitutions: []
+    )
+      normalized_id = id.to_sym
+
+      return populate_context! if normalized_id == context_id
+
+      Lutaml::Model::GlobalContext.unregister_context(normalized_id) if Lutaml::Model::GlobalContext.context(normalized_id)
+      create_type_context(
+        id: normalized_id,
+        registry: registry || Lutaml::Model::TypeRegistry.new,
+        fallback_to: normalize_fallbacks(fallback_to),
+        substitutions: substitutions,
+      )
+    end
+
+    def populate_context!
+      Lutaml::Model::GlobalContext.unregister_context(context_id) if context
+      register_models_in(base_type_context)
+    end
+
+    def register_model(klass, id:)
+      normalized_id = id.to_sym
+      registered_models[normalized_id] = klass
+      (context || populate_base_context).registry.register(normalized_id, klass)
+      klass
+    end
+
+    def resolve_model(id)
+      Lutaml::Model::GlobalContext.resolve_type(id, context_id)
+    end
+
+    private
+
+    def populate_base_context
+      base_type_context
+    end
+
+    def create_type_context(id:, registry:, fallback_to:, substitutions: [])
+      Lutaml::Model::GlobalContext.create_context(
+        id: id,
+        registry: registry,
+        fallback_to: fallback_to,
+        substitutions: substitutions,
+      ).tap do
+        Lutaml::Model::GlobalContext.clear_caches
+      end
+    end
+
+    def base_type_context
+      create_type_context(
+        id: context_id,
+        registry: Lutaml::Model::TypeRegistry.new,
+        fallback_to: [:default],
+      )
+    end
+
+    def register_models_in(type_context)
+      registered_models.each do |model_id, klass|
+        type_context.registry.register(model_id, klass)
+      end
+
+      Lutaml::Model::GlobalContext.clear_caches
+      type_context
+    end
+
+    def normalize_fallbacks(fallback_to)
+      Array(fallback_to).map(&:to_sym)
+    end
+
+    def registered_models
+      @registered_models ||= {}
+    end
+  end
+end

data/lib/glossarist/dataset_validator.rb

@@ -2,6 +2,10 @@
 
 module Glossarist
   class DatasetValidator
+    def initialize(on_progress: nil)
+      @on_progress = on_progress
+    end
+
     def validate(path, strict: false, reference_path: nil)
       if File.extname(path).downcase == ".gcr"
         validate_gcr(path, reference_path: reference_path)
@@ -13,7 +17,7 @@ module Glossarist
     private
 
     def validate_gcr(path, reference_path: nil)
-      result = GcrValidator.new.validate(path)
+      result = GcrValidator.new(on_progress: @on_progress).validate(path)
 
       if reference_path
         ref_result = validate_gcr_cross_references(path, reference_path)
@@ -24,7 +28,7 @@ module Glossarist
     end
 
     def validate_directory(path, reference_path: nil)
-      result = ConceptValidator.new(path).validate_all
+      result = ConceptValidator.new(path, on_progress: @on_progress).validate_all
 
       if reference_path
         ref_result = validate_directory_cross_references(path, reference_path)
@@ -38,7 +42,7 @@ module Glossarist
       extractor = ReferenceExtractor.new
       resolver = build_resolver(reference_path)
       pkg = GcrPackage.load(path)
-      uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
+      uri_prefix = pkg.metadata&.uri_prefix || pkg.metadata&.shortname
       resolver.register_self(pkg.concepts)
       resolver.register_package(pkg, uri_prefix: uri_prefix)
       resolver.validate_all(pkg, extractor: extractor)
@@ -56,7 +60,7 @@ module Glossarist
       resolver = ReferenceResolver.new
       Dir.glob(File.join(reference_path, "*.gcr")).each do |gcr_path|
         pkg = GcrPackage.load(gcr_path)
-        uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
+        uri_prefix = pkg.metadata&.uri_prefix || pkg.metadata&.shortname
         resolver.register_package(pkg, uri_prefix: uri_prefix)
       end
       resolver

data/lib/glossarist/designation/abbreviation.rb

@@ -1,7 +1,6 @@
 module Glossarist
   module Designation
     class Abbreviation < Expression
-      attribute :international, :boolean
       attribute :type, :string, default: -> { "abbreviation" }
 
       Glossarist::GlossaryDefinition::ABBREVIATION_TYPES.each do |name|
@@ -9,7 +8,6 @@ module Glossarist
       end
 
       key_value do
-        map :international, to: :international
         map :type, to: :type, render_default: true
         Glossarist::GlossaryDefinition::ABBREVIATION_TYPES.each do |name|
           map name.to_sym, to: name.to_sym

data/lib/glossarist/designation/base.rb

@@ -6,12 +6,30 @@ module Glossarist
       attribute :normative_status, :string,
                 values: Glossarist::GlossaryDefinition::DESIGNATION_BASE_NORMATIVE_STATUSES
       attribute :type, :string
+      attribute :language, :string
+      attribute :script, :string
+      attribute :system, :string
+      attribute :international, :boolean
+      attribute :absent, :boolean
+      attribute :pronunciation, Pronunciation, collection: true
+      attribute :sources, ConceptSource, collection: true
+      attribute :term_type, :string
+      attribute :related, RelatedConcept, collection: true
 
       key_value do
         map :type, to: :type
         map %i[normative_status normativeStatus], to: :normative_status
         map %i[geographical_area geographicalArea], to: :geographical_area
         map :designation, to: :designation
+        map :language, to: :language
+        map :script, to: :script
+        map :system, to: :system
+        map :international, to: :international
+        map :absent, to: :absent
+        map :pronunciation, to: :pronunciation
+        map :sources, to: :sources
+        map %i[term_type termType], to: :term_type
+        map :related, to: :related
       end
 
       def self.of_yaml(hash, options = {})
@@ -34,7 +52,9 @@ module Glossarist
       end
 
       def self.infer_designation_type(hash)
-        if hash["international"] || hash["abbreviation_type"]
+        if hash["abbreviation_type"]
+          "abbreviation"
+        elsif hash["international"]
           "symbol"
         else
           "expression"

data/lib/glossarist/designation/expression.rb

@@ -5,6 +5,7 @@ module Glossarist
     class Expression < Base
       attribute :prefix, :string
       attribute :usage_info, :string
+      attribute :field_of_application, :string
 
       attribute :gender, :string
       attribute :plurality, :string
@@ -16,6 +17,8 @@ module Glossarist
         map :type, to: :type, render_default: true
         map :prefix, to: :prefix
         map %i[usage_info usageInfo], to: :usage_info
+        map %i[field_of_application fieldOfApplication],
+            to: :field_of_application
         map %i[grammar_info grammarInfo], to: :grammar_info
       end
 

data/lib/glossarist/designation/letter_symbol.rb

@@ -2,13 +2,9 @@ module Glossarist
   module Designation
     class LetterSymbol < Symbol
       attribute :text, :string
-      attribute :language, :string
-      attribute :script, :string
 
       key_value do
         map :text, to: :text
-        map :language, to: :language
-        map :script, to: :script
       end
 
       def self.of_yaml(hash, options = {})

data/lib/glossarist/designation/prefix.rb

@@ -0,0 +1,17 @@
+module Glossarist
+  module Designation
+    class Prefix < Base
+      attribute :type, :string, default: -> { "prefix" }
+
+      key_value do
+        map :type, to: :type, render_default: true
+      end
+
+      def self.of_yaml(hash, options = {})
+        hash["type"] = "prefix" unless hash["type"]
+
+        super
+      end
+    end
+  end
+end