glossarist 2.5.0 → 2.5.2

data/lib/glossarist/cli/upgrade_command.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class CLI
+    class UpgradeCommand
+      def initialize(source_dir, options)
+        @source_dir = source_dir
+        @options = options
+      end
+
+      def run
+        result = SchemaMigration.upgrade_directory(
+          @source_dir,
+          output: @options[:output],
+          target_version: @options[:target_version],
+          cross_references: @options[:cross_references],
+          dry_run: @options[:dry_run],
+        )
+        report(result)
+      rescue ArgumentError => e
+        warn "Error: #{e.message}"
+        exit 1
+      end
+
+      private
+
+      def report(result)
+        puts "Upgraded #{result[:count]} concepts " \
+             "from schema v#{result[:source_version]} to v#{result[:target_version]}"
+        puts "Output: #{result[:output]}"
+      end
+    end
+  end
+end

data/lib/glossarist/cli/validate_command.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class CLI
+    class ValidateCommand
+      def initialize(path, options)
+        @path = path
+        @options = options
+      end
+
+      def run
+        result = DatasetValidator.new.validate(
+          @path,
+          strict: @options[:strict],
+          reference_path: @options[:reference_path],
+        )
+        report(result)
+        exit_code = result.errors.any? || (@options[:strict] && result.warnings.any?) ? 1 : 0
+        exit(exit_code) unless exit_code.zero?
+      end
+
+      private
+
+      def report(result)
+        case @options[:format]
+        when "json"
+          require "json"
+          puts JSON.pretty_generate(result.to_h)
+        when "yaml"
+          require "yaml"
+          puts YAML.dump(result.to_h)
+        else
+          report_text(result)
+        end
+      end
+
+      def report_text(result)
+        if result.valid?
+          puts "Valid."
+        else
+          puts "Invalid."
+          result.errors.each { |e| puts "  ERROR: #{e}" }
+        end
+
+        if result.warnings.any?
+          result.warnings.each do |w|
+            puts "  WARNING: #{w}"
+          end
+        end
+
+        total = result.errors.length + result.warnings.length
+        puts "#{total} issue(s) found."
+      end
+    end
+  end
+end

data/lib/glossarist/cli.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module Glossarist
+  class CLI < Thor
+    desc "generate_latex", "Convert Concepts to Latex format"
+
+    option :concepts_path, aliases: :p, required: true,
+                           desc: "Path to yaml concepts directory"
+    option :latex_concepts, aliases: :l,
+                            desc: "File path having list of concepts that should be converted to LATEX format. If not provided all the concepts will be converted to the latex format"
+    option :output_file, aliases: :o,
+                         desc: "Output file path. By default the output will pe printed to the console"
+    option :extra_attributes, aliases: :e, type: :array,
+                              desc: "List of extra attributes that are not in standard Glossarist Concept model"
+    def generate_latex
+      assets = []
+
+      if options[:extra_attributes]
+        Glossarist.configure do |config|
+          config.register_extension_attributes(options[:extra_attributes])
+        end
+      end
+
+      concept_set = Glossarist::ConceptSet.new(options[:concepts_path], assets)
+      latex_str = concept_set.to_latex(options[:latex_concepts])
+      output_latex(latex_str)
+    end
+
+    desc "upgrade SOURCE_DIR", "Upgrade dataset to current schema version"
+    option :output, aliases: :o, required: true,
+                    desc: "Output directory or .gcr file path"
+    option :target_version, type: :string, default: Glossarist::SchemaMigration::CURRENT_SCHEMA_VERSION,
+                            desc: "Target schema version (default: current)"
+    option :cross_references, type: :string,
+                              desc: "Path to datasets.yml for cross-reference maps"
+    option :dry_run, type: :boolean, default: false,
+                     desc: "Show what would change without writing"
+    def upgrade(source_dir)
+      require_relative "cli/upgrade_command"
+      Glossarist::CLI::UpgradeCommand.new(source_dir, options).run
+    end
+
+    desc "package DIR", "Create a .gcr ZIP archive from a schema v1 dataset"
+    option :output, aliases: :o, required: true,
+                    desc: "Output .gcr file path"
+    option :shortname, type: :string, required: true,
+                       desc: "Machine-readable dataset ID"
+    option :version, type: :string, required: true,
+                     desc: "Semantic version (e.g. 1.0.0)"
+    option :uri_prefix, type: :string,
+                        desc: "URI namespace prefix (e.g. urn:iec:std:iec:60050)"
+    option :title, type: :string, desc: "Dataset title"
+    option :description, type: :string, desc: "Dataset description"
+    option :owner, type: :string, desc: "Dataset owner"
+    option :register_yaml, type: :string,
+                           desc: "Path to register.yaml to include in package"
+    option :tags, type: :array, desc: "Tags for the dataset"
+    def package(dir)
+      require_relative "cli/package_command"
+      Glossarist::CLI::PackageCommand.new(dir, options).run
+    end
+
+    desc "validate PATH",
+         "Validate dataset directory or .gcr for schema compliance"
+    option :strict, type: :boolean, default: false,
+                    desc: "Treat warnings as errors"
+    option :format, type: :string, default: "text",
+                    enum: %w[text json yaml],
+                    desc: "Output format for validation results"
+    option :reference_path, type: :string,
+                            desc: "Path to directory of .gcr files for cross-dataset reference validation"
+    def validate(path)
+      require_relative "cli/validate_command"
+      Glossarist::CLI::ValidateCommand.new(path, options).run
+    end
+
+    def method_missing(*args)
+      warn "No method found named: #{args[0]}"
+      warn "Run with `--help` or `-h` to see available options"
+      exit 1
+    end
+
+    def respond_to_missing?
+      true
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+
+    private
+
+    def output_latex(latex_str)
+      output_file_path = options[:output_file]
+
+      if output_file_path
+        File.open(output_file_path, "w") { |file| file.puts latex_str }
+      else
+        puts latex_str
+      end
+    end
+  end
+end

data/lib/glossarist/collection_config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class CollectionConfig < Lutaml::Model::Serializable
+    attribute :packages, :hash, collection: true, default: -> { [] }
+    attribute :routes, :hash, collection: true, default: -> { [] }
+    attribute :remotes, :hash, collection: true, default: -> { [] }
+
+    yaml do
+      map :packages, to: :packages
+      map :routes, to: :routes
+      map :remote, to: :remotes
+    end
+
+    def self.from_file(path)
+      return nil unless File.exist?(path)
+
+      from_yaml(File.read(path))
+    rescue Psych::SyntaxError, Lutaml::Model::InvalidFormatError
+      nil
+    end
+  end
+end

data/lib/glossarist/collections.rb

@@ -2,13 +2,20 @@
 
 module Glossarist
   module Collections
-    autoload :AssetCollection,               "glossarist/collections/asset_collection"
-    autoload :BibliographyCollection,        "glossarist/collections/bibliography_collection"
-    autoload :Collection,                    "glossarist/collections/collection"
-    autoload :ConceptSourceCollection,       "glossarist/collections/concept_source_collection"
-    autoload :DesignationCollection,         "glossarist/collections/designation_collection"
-    autoload :DetailedDefinitionCollection,  "glossarist/collections/detailed_definition_collection"
-    autoload :LocalizationCollection,        "glossarist/collections/localization_collection"
-    autoload :TypedCollection,               "glossarist/collections/typed_collection"
+    autoload :AssetCollection,
+             "glossarist/collections/asset_collection"
+    autoload :BibliographyCollection,
+             "glossarist/collections/bibliography_collection"
+    autoload :Collection, "glossarist/collections/collection"
+    autoload :ConceptSourceCollection,
+             "glossarist/collections/concept_source_collection"
+    autoload :DesignationCollection,
+             "glossarist/collections/designation_collection"
+    autoload :DetailedDefinitionCollection,
+             "glossarist/collections/detailed_definition_collection"
+    autoload :LocalizationCollection,
+             "glossarist/collections/localization_collection"
+    autoload :TypedCollection,
+             "glossarist/collections/typed_collection"
   end
 end

data/lib/glossarist/concept.rb

@@ -11,7 +11,7 @@ module Glossarist
     attribute :extension_attributes, :hash
     attribute :termid, :string
 
-    yaml do
+    key_value do
       map :data, to: :data
       map :termid, to: :termid
       map :subject, to: :subject

data/lib/glossarist/concept_collector.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptCollector
+    def self.collect(dir)
+      dir = File.expand_path(dir)
+      unless File.directory?(dir)
+        raise ArgumentError, "#{dir} is not a directory"
+      end
+
+      if v2_concepts?(dir)
+        collect_v2_concepts(dir)
+      elsif managed_concepts?(dir)
+        collect_managed_concepts(dir)
+      elsif v1_concepts?(dir)
+        collect_v1_concepts(dir)
+      else
+        []
+      end
+    end
+
+    def self.each_concept(dir, &block)
+      dir = File.expand_path(dir)
+      unless File.directory?(dir)
+        raise ArgumentError, "#{dir} is not a directory"
+      end
+      return enum_for(:each_concept, dir) unless block
+
+      if v2_concepts?(dir)
+        each_v2_concept(dir, &block)
+      elsif managed_concepts?(dir)
+        each_managed_concept(dir, &block)
+      elsif v1_concepts?(dir)
+        each_v1_concept(dir, &block)
+      end
+    end
+
+    class << self
+      private
+
+      def v1_concepts?(dir)
+        concepts_dir = File.join(dir, "concepts")
+        File.directory?(concepts_dir) &&
+          Dir.glob(File.join(concepts_dir, "*.yaml")).any? do |f|
+            V1::Concept.from_file(f)&.termid?
+          end
+      end
+
+      def v2_concepts?(dir)
+        File.directory?(File.join(dir, "geolexica-v2"))
+      end
+
+      def managed_concepts?(dir)
+        concept_dir = File.join(dir, "concepts", "concept")
+        File.directory?(concept_dir) &&
+          Dir.glob(File.join(concept_dir, "*.yaml")).any?
+      end
+
+      def collect_v1_concepts(dir)
+        concepts = []
+        each_v1_concept(dir) { |mc| concepts << mc }
+        concepts
+      end
+
+      def each_v1_concept(dir)
+        concepts_dir = File.join(dir, "concepts")
+        files = Dir.glob(File.join(concepts_dir, "*.yaml"))
+        files.each do |file|
+          v1 = V1::Concept.from_file(file)
+          next unless v1
+
+          yield v1.to_managed_concept
+        end
+      end
+
+      def collect_v2_concepts(dir)
+        v2_dir = File.join(dir, "geolexica-v2")
+        if File.directory?(File.join(v2_dir, "concepts"))
+          collect_managed_concepts(v2_dir)
+        else
+          collect_grouped_v2_concepts(v2_dir)
+        end
+      end
+
+      def each_v2_concept(dir, &block)
+        v2_dir = File.join(dir, "geolexica-v2")
+        if File.directory?(File.join(v2_dir, "concepts"))
+          each_managed_concept(v2_dir, &block)
+        else
+          each_grouped_v2_concept(v2_dir, &block)
+        end
+      end
+
+      def each_grouped_v2_concept(v2_dir, &block)
+        collection = ManagedConceptCollection.new
+        manager = ConceptManager.new(path: v2_dir)
+        manager.load_from_files(collection: collection)
+        collection.each(&block)
+      end
+
+      def collect_grouped_v2_concepts(v2_dir)
+        collection = ManagedConceptCollection.new
+        manager = ConceptManager.new(path: v2_dir)
+        manager.load_from_files(collection: collection)
+        collection.to_a
+      end
+
+      def collect_managed_concepts(dir) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        concepts = []
+        each_managed_concept(dir) { |mc| concepts << mc }
+        concepts
+      end
+
+      def each_managed_concept(dir) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        concepts_dir = File.join(dir, "concepts")
+        concept_files = Dir.glob(File.join(concepts_dir, "concept", "*.yaml"))
+        return if concept_files.empty?
+
+        lc_dir = find_localized_concepts_dir(concepts_dir)
+        lc_index = build_lc_index(lc_dir) if lc_dir
+
+        concept_files.each do |f|
+          mc = ManagedConcept.from_yaml(File.read(f))
+          next unless mc.data&.id
+
+          lc_map = mc.data.localized_concepts || {}
+          lc_map.each_value do |uuid|
+            lc_file = lc_index ? lc_index[uuid] : nil
+            next unless lc_file
+
+            l10n = LocalizedConcept.from_yaml(File.read(lc_file))
+            mc.add_localization(l10n)
+          end
+
+          yield mc
+        end
+      end
+
+      def build_lc_index(lc_dir)
+        Dir.glob(File.join(lc_dir, "*.{yaml,yml}"))
+          .to_h { |f| [File.basename(f, ".*"), f] }
+      end
+
+      def find_localized_concepts_dir(concepts_dir)
+        %w[localized_concept localized-concept].each do |name|
+          d = File.join(concepts_dir, name)
+          return d if File.directory?(d)
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/glossarist/concept_data.rb

@@ -20,6 +20,7 @@ module Glossarist
               initialize_empty: true
     attribute :terms, Designation::Base, collection: true
     attribute :related, RelatedConcept, collection: true
+    attribute :references, ConceptReference, collection: true
     attribute :domain, :string
     attribute :review_date, :date_time
     attribute :review_decision_date, :date_time
@@ -31,7 +32,7 @@ module Glossarist
     attribute :language_code, :string, pattern: /^.{3}$/
     attribute :entry_status, :string
 
-    yaml do
+    key_value do
       map :dates, to: :dates
       map :definition, to: :definition, value_map: { to: { empty: :empty } }
       map :examples, to: :examples, value_map: { to: { empty: :empty } }
@@ -44,6 +45,7 @@ module Glossarist
       map :terms, to: :terms,
                   with: { from: :terms_from_yaml, to: :terms_to_yaml }
       map :related, to: :related
+      map :references, to: :references
       map :domain, to: :domain
       map %i[language_code languageCode], to: :language_code
       map %i[entry_status entryStatus], to: :entry_status

data/lib/glossarist/concept_date.rb

@@ -8,7 +8,7 @@ module Glossarist
     attribute :type, :string,
               values: Glossarist::GlossaryDefinition::CONCEPT_DATE_TYPES
 
-    yaml do
+    key_value do
       map :date, to: :date
       map :type, to: :type
     end

data/lib/glossarist/concept_document.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptDocument < Lutaml::Model::Serializable
+    attribute :concept, ManagedConcept
+    attribute :localizations, LocalizedConcept, collection: true
+
+    yamls do
+      sequence do
+        map_document 0, to: :concept, type: ManagedConcept
+        map_document 1.., to: :localizations, type: LocalizedConcept,
+                                           collection: true
+      end
+    end
+
+    def self.from_managed_concept(managed_concept)
+      new(
+        concept: managed_concept,
+        localizations: managed_concept.localizations&.values || [],
+      )
+    end
+
+    def to_managed_concept
+      mc = concept
+      localizations.each { |l10n| mc.add_localization(l10n) }
+      mc
+    end
+  end
+end

data/lib/glossarist/concept_enricher.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptEnricher
+    def inject_references(concepts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+      extractor = ReferenceExtractor.new
+
+      concepts.each do |mc|
+        mc.localizations.each do |l10n|
+          refs = extractor.extract_from_localized_concept(l10n)
+          next if refs.empty?
+
+          existing = l10n.data.references || []
+          seen_keys = existing.to_set { |r| [r.source, r.concept_id] }
+
+          refs.each do |ref|
+            key = [ref.source, ref.concept_id]
+            next if seen_keys.include?(key)
+
+            seen_keys.add(key)
+            existing << ref
+          end
+          l10n.data.references = existing
+        end
+      end
+    end
+
+    def apply_uri_template(concepts, template)
+      concepts.each do |mc|
+        mc.data.uri = template.sub("{id}", mc.data.id.to_s)
+      end
+    end
+  end
+end

data/lib/glossarist/concept_manager.rb

@@ -3,7 +3,7 @@ module Glossarist
     attribute :path, :string
     attribute :localized_concepts_path, :string
 
-    yaml do
+    key_value do
       map :path, to: :path
       map %i[localized_concepts_path localizedConceptsPath],
           to: :localized_concepts_path
@@ -33,62 +33,43 @@ module Glossarist
       end
     end
 
-    def group_concept_hashes(mixed_hashes)
-      concept_hashes = mixed_hashes.select do |concept_hash|
-        !concept_hash["data"]["localized_concepts"].nil? ||
-          !concept_hash["data"]["localizedConcepts"].nil?
+    def load_concept_from_file(filename) # rubocop:disable Metrics/CyclomaticComplexity
+      raw = File.read(filename, encoding: "utf-8")
+      doc = ConceptDocument.from_yamls(raw)
+      concept = doc.concept
+      unless concept
+        raise Glossarist::ParseError.new(filename: filename)
       end
 
-      localized_concept_hashes = mixed_hashes.select do |concept_hash|
-        concept_hash["data"]["localized_concepts"].nil? &&
-          concept_hash["data"]["localizedConcepts"].nil?
-      end
-
-      [concept_hashes, localized_concept_hashes]
-    end
-
-    def load_concept_from_file(filename)
-      mixed_hashes = YAML.load_stream(File.read(filename, encoding: "utf-8"))
-      concepts = []
-
-      concept_hashes, localized_concept_hashes =
-        group_concept_hashes(mixed_hashes)
-
-      concept_hashes.each do |concept_hash|
-        concept_hash["uuid"] = concept_hash["id"] ||
-          File.basename(filename, ".*")
-        concept = Config.class_for(:managed_concept).of_yaml(concept_hash)
+      concept_uuid = concept.identifier || concept.data&.id || File.basename(
+        filename, ".*"
+      )
+      concept.instance_variable_set(:@uuid, concept_uuid)
 
-        concept.data.localized_concepts.each_value do |id|
-          localized_concept =
-            load_localized_concept(id, localized_concept_hashes)
-          concept.add_l10n(localized_concept)
-        end
-
-        concepts << concept
+      concept.data.localized_concepts.each_value do |id|
+        localized_concept = load_localized_concept(id, doc.localizations)
+        concept.add_l10n(localized_concept)
       end
 
-      concepts
+      [concept]
     rescue Psych::SyntaxError => e
       raise Glossarist::ParseError.new(filename: filename, line: e.line)
     end
 
-    def load_localized_concept(id, localized_concept_hashes = [])
-      {}
-
-      concept_hash = if localized_concept_hashes.empty?
-                       Psych.safe_load(
-                         File.read(localized_concept_path(id), encoding: "utf-8"),
-                         permitted_classes: [Date, Time],
-                       )
-                     else
-                       localized_concept_hashes.find do |hash|
-                         hash["id"] == id
-                       end
-                     end
-
-      concept_hash["uuid"] = id
-      Config.class_for(:localized_concept).of_yaml(concept_hash)
+    def load_localized_concept(id, inline_localizations = nil)
+      if inline_localizations
+        l10n = inline_localizations.find { |l| l.id == id }
+        if l10n
+          l10n.instance_variable_set(:@uuid, id)
+          return l10n
+        end
+      end
+
+      l10n = LocalizedConcept.from_yaml(
+        File.read(localized_concept_path(id), encoding: "utf-8"),
+      )
+      l10n.instance_variable_set(:@uuid, id)
+      l10n
     rescue Psych::SyntaxError => e
       raise Glossarist::ParseError.new(filename: filename, line: e.line)
     end
@@ -107,7 +88,8 @@ module Glossarist
 
       concept.localized_concepts.each do |lang, uuid|
         filename = File.join(localized_concept_dir, "#{uuid}.yaml")
-        File.write(filename, concept.localization(lang).to_yaml, encoding: "utf-8")
+        File.write(filename, concept.localization(lang).to_yaml,
+                   encoding: "utf-8")
       end
     end
 

data/lib/glossarist/concept_reference.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptReference < Lutaml::Model::Serializable
+    attribute :term, :string
+    attribute :concept_id, :string
+    attribute :source, :string
+    attribute :ref_type, :string
+
+    key_value do
+      map :term, to: :term
+      map :concept_id, to: :concept_id
+      map :source, to: :source
+      map :ref_type, to: :ref_type
+    end
+
+    def local?
+      %w[local designation].include?(ref_type) ||
+        (ref_type.nil? && (source.nil? || source.empty?))
+    end
+
+    def external?
+      !local?
+    end
+
+    def to_urn
+      return nil unless external?
+      return nil unless source && concept_id
+
+      case source
+      when /\Aurn:iec/ then "#{source}-#{concept_id}"
+      when /\Aurn:iso/ then "#{source}:term:#{concept_id}"
+      else "#{source}/#{concept_id}"
+      end
+    end
+
+    def to_gcr_hash
+      h = { "term" => term }
+      h["concept_id"] = concept_id if concept_id
+      h["source"] = source if source
+      h["ref_type"] = ref_type if ref_type
+      h.compact
+    end
+  end
+end

data/lib/glossarist/concept_source.rb

@@ -7,7 +7,7 @@ module Glossarist
     attribute :origin, Citation
     attribute :modification, :string
 
-    yaml do
+    key_value do
       # TODO: change to `map [:ref, :origin], to: :origin
       #       when multiple key mapping is supported in lutaml-model
       map :origin, to: :origin