glossarist 2.6.5 → 2.6.7

data/config.yml

@@ -26,23 +26,90 @@ designation:
   base:
     normative_status:
       - preferred
-      - deprecated
       - admitted
+      - deprecated
+      - superseded
       - <símbolo> # in iev-data => '<symbol>'
       - 티에스 # in iev-data => translates to 'TS' I think it is a synonym and not status.
       - prąd startowy # in iev-data => 'starting current' I think it is a synonym and not status.
+  relationship_type:
+    # Term-level (designation-to-designation within the same concept entry)
+    - abbreviated_form_for
+    - short_form_for
 
 related_concept:
   type:
+    # Lifecycle (ISO 10241-1)
     - deprecates
     - supersedes
     - superseded_by
+    # Hierarchical (ISO 10241-1 / ISO 25964)
     - narrower
     - broader
+    # Hierarchical sub-types — ISO 25964 generic (BTG/NTG)
+    - broader_generic
+    - narrower_generic
+    # Hierarchical sub-types — ISO 25964 partitive (BTP/NTP)
+    - broader_partitive
+    - narrower_partitive
+    # Hierarchical sub-types — ISO 25964 instantial (BTI/NTI)
+    - broader_instantial
+    - narrower_instantial
+    # Equivalence (ISO 10241-1 / ISO 25964 exactMatch / SKOS)
     - equivalent
+    # Approximate equivalence (ISO 25964 closeMatch / SKOS)
+    - close_match
+    # Cross-vocabulary mapping (SKOS)
+    - broad_match
+    - narrow_match
+    - related_match
+    # Comparative (ISO 10241-1)
     - compare
     - contrast
+    # Associative (ISO 10241-1 / ISO 25964 RT / TBX crossReference)
     - see
+    # Associative sub-types (ISO 25964 / TBX)
+    - related_concept
+    - related_concept_broader
+    - related_concept_narrower
+    # Associative — spatial/temporal (ISO 25964 / TBX)
+    - sequentially_related_concept
+    - spatially_related_concept
+    - temporally_related_concept
+    # Lexical (ISO 12620 / TBX)
+    - homograph
+    - false_friend
+
+iso12620:
+  term_type:
+    # ISO 12620 term type classification for designations.
+    # Orthographic / structural:
+    - abbreviation
+    - acronym
+    - clipped_term
+    - full_form
+    - initialism
+    - short_form
+    - transliterated_form
+    - transcribed_form
+    - variant
+    # Symbolic / formulaic:
+    - equation
+    - formula
+    - logical_expression
+    - symbol
+    # Usage / provenance:
+    - common_name
+    - entry_term
+    - internationalism
+    - international_scientific_term
+    - part_number
+    - phraseological_unit
+    - shortcut
+    - sku
+    - standard_text
+    - synonym
+    - synonymous_phrase
 
 abbreviation:
   type:

data/glossarist.gemspec

@@ -32,9 +32,11 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "lutaml-model", "~> 0.8.5"
+  spec.add_dependency "paint", "~> 2.3"
   spec.add_dependency "relaton", ">= 2.0.0", "< 3"
   spec.add_dependency "rubyzip", ">= 2.3", "< 3"
   spec.add_dependency "sts", "~> 0.5.6"
+  spec.add_dependency "table_tennis", "~> 0.0"
   spec.add_dependency "tbx", "~> 0.1"
   spec.add_dependency "thor"
 end

data/lib/glossarist/citation.rb

@@ -1,122 +1,41 @@
+# frozen_string_literal: true
+
 module Glossarist
   class Citation < Lutaml::Model::Serializable
-    # Unstructured (plain text) reference.
-    # @return [String]
-    attribute :text, :string
-
-    # Source in structured reference.
-    # @return [String]
-    attribute :source, :string
-
-    # Document ID in structured reference.
-    # @return [String]
-    attribute :id, :string
-
-    # Document version in structured reference.
-    # @return [String]
-    attribute :version, :string
+    class Ref < Lutaml::Model::Serializable
+      attribute :source, :string
+      attribute :id, :string
+      attribute :version, :string
+
+      key_value do
+        map :source, to: :source
+        map :id, to: :id
+        map :version, to: :version
+      end
+    end
 
-    # @return [String]
-    # Referred locality of the document.
+    attribute :ref, Ref
     attribute :locality, Locality
-
-    # Link to document.
-    # @return [String]
     attribute :link, :string
-
-    # Original ref text before parsing.
-    # @return [String]
-    # @note This attribute is likely to be removed or reworked in future.
-    #   It is arguably not relevant to Glossarist itself.
     attribute :original, :string
-
-    attribute :ref, :string
-
     attribute :custom_locality, CustomLocality, collection: true
 
     key_value do
-      map :id, to: :id, with: { from: :id_from_yaml, to: :id_to_yaml }
-      map :text, to: :text, with: { from: :text_from_yaml, to: :text_to_yaml }
-      map :source, to: :source,
-                   with: { from: :source_from_yaml, to: :source_to_yaml }
-      map :version, to: :version,
-                    with: { from: :version_from_yaml, to: :version_to_yaml }
-      map :ref, to: :ref, with: { from: :ref_from_yaml, to: :ref_to_yaml }
+      map :ref, to: :ref
       map %i[clause locality],
           to: :locality,
-          with: { from: :clause_from_yaml, to: :clause_to_yaml }
+          with: { from: :locality_from_yaml, to: :locality_to_yaml }
       map :link, to: :link
       map :original, to: :original
       map %i[custom_locality customLocality], to: :custom_locality
     end
 
-    def ref_from_yaml(model, value)
-      model.ref = value
-    end
-
-    def ref_to_yaml(model, doc)
-      doc["ref"] = if model.structured?
-                     ref_hash(model)
-                   else
-                     model.text
-                   end
-    end
-
-    def id_from_yaml(model, value)
-      model.id = value
-    end
-
-    def id_to_yaml(_model, _doc)
-      # skip, will be handled in ref
-    end
-
-    def text_from_yaml(model, value)
-      model.text = value
-    end
-
-    def text_to_yaml(_model, _doc)
-      # skip, will be handled in ref
+    def label
+      parts = [ref&.source, ref&.id].compact
+      parts.empty? ? nil : parts.join(" ")
     end
 
-    def source_from_yaml(model, value)
-      model.source = value
-    end
-
-    def source_to_yaml(_model, _doc)
-      # skip, will be handled in ref
-    end
-
-    def version_from_yaml(model, value)
-      model.version = value
-    end
-
-    def version_to_yaml(_model, _doc)
-      # skip, will be handled in ref
-    end
-
-    def ref_hash(model = self)
-      {
-        "source" => model.source,
-        "id" => model.id,
-        "version" => model.version,
-      }.compact
-    end
-
-    def ref=(ref)
-      if ref.is_a?(Hash)
-        @source = ref["source"]
-        @id = ref["id"]
-        @version = ref["version"]
-      else
-        @text = ref
-      end
-    end
-
-    def clause_from_yaml(model, value) # rubocop:disable Metrics/AbcSize
-      # accepts old format like
-      # clause: "11"
-      # or new format like
-      # locality: { type: "clause", reference_from: "11", reference_to: "12" }
+    def locality_from_yaml(model, value)
       locality = Locality.new
 
       if value.is_a?(Hash)
@@ -132,29 +51,13 @@ module Glossarist
       model.locality = locality
     end
 
-    def clause_to_yaml(model, doc) # rubocop:disable Metrics/AbcSize
-      if model.locality
-        doc["locality"] = {}
-        doc["locality"]["type"] = model.locality.type
-
-        if model.locality.reference_from
-          doc["locality"]["reference_from"] = model.locality.reference_from
-        end
-
-        if model.locality.reference_to
-          doc["locality"]["reference_to"] = model.locality.reference_to
-        end
-      end
-    end
-
-    def plain?
-      (source && id && version).nil?
-    end
+    def locality_to_yaml(model, doc)
+      return unless model.locality
 
-    # Whether it is a structured ref.
-    # @return [Boolean]
-    def structured?
-      !plain?
+      doc["locality"] = {}
+      doc["locality"]["type"] = model.locality.type
+      doc["locality"]["reference_from"] = model.locality.reference_from if model.locality.reference_from
+      doc["locality"]["reference_to"] = model.locality.reference_to if model.locality.reference_to
     end
   end
 end

data/lib/glossarist/cli/compare_command.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "paint"
+
+module Glossarist
+  class CLI
+    class CompareCommand
+      def initialize(new_path, old_path, options)
+        @new_path = new_path
+        @old_path = old_path
+        @options = options
+      end
+
+      def run
+        new_concepts = ConceptCollector.collect(@new_path)
+        old_concepts = ConceptCollector.collect(@old_path)
+
+        result = ConceptComparator.new(
+          new_concepts: new_concepts,
+          old_concepts: old_concepts,
+        ).compare(show_diffs: !@options[:no_diffs])
+
+        report(result)
+      end
+
+      private
+
+      def report(result)
+        case @options[:format]
+        when "json"
+          output result.to_json
+        when "yaml"
+          output result.to_yaml
+        else
+          print_text_report(result)
+        end
+      end
+
+      def print_text_report(result)
+        puts
+        puts Paint["Concept Comparison", :bold]
+        puts
+
+        print_counts(result)
+        print_new_only(result)
+        print_old_only(result)
+        print_similarity(result)
+
+        puts
+      end
+
+      def print_counts(result)
+        puts "  #{Paint['New:', :bold]} #{result.new_count} concepts"
+        puts "  #{Paint['Old:', :bold]} #{result.old_count} concepts"
+        puts "  #{Paint['Matched:', :bold]} #{result.matched.length}"
+        puts "  #{Paint['New only:', :bold]} #{result.new_only.length}"
+        puts "  #{Paint['Old only:', :bold]} #{result.old_only.length}"
+      end
+
+      def print_new_only(result)
+        return unless result.new_only.any?
+
+        puts
+        puts "  #{Paint['New concepts (not in old):', :green, :bold]}"
+        result.new_only.each { |id| puts "    + #{id}" }
+      end
+
+      def print_old_only(result)
+        return unless result.old_only.any?
+
+        puts
+        puts "  #{Paint['Removed concepts (not in new):', :red, :bold]}"
+        result.old_only.each { |id| puts "    - #{id}" }
+      end
+
+      def print_similarity(result)
+        return unless result.diffs.any?
+
+        puts
+        puts "  #{Paint['Per-concept similarity:', :bold]}"
+        result.diffs.each do |diff|
+          color = similarity_color(diff.similarity)
+          puts "    #{diff.concept_id}: #{Paint["#{diff.similarity}%", color]}"
+        end
+      end
+
+      def similarity_color(value)
+        if value >= 100
+          :green
+        elsif value >= 90
+          :yellow
+        else
+          :red
+        end
+      end
+
+      def output(content)
+        if @options[:report]
+          File.write(@options[:report], content)
+        else
+          puts content
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/cli/export_command.rb

@@ -48,8 +48,8 @@ module Glossarist
       end
 
       def resolve_metadata_from_package(package)
-        @options[:shortname] ||= package.metadata["shortname"]
-        @options[:uri_prefix] ||= package.metadata["uri_prefix"]
+        @options[:shortname] ||= package.metadata.shortname
+        @options[:uri_prefix] ||= package.metadata.uri_prefix
       end
 
       def resolve_shortname
@@ -73,17 +73,15 @@ module Glossarist
       end
 
       def export_jsonld(concepts, name, output_dir)
-        require "glossarist/transforms/concept_to_skos_transform"
-        vocab = Transforms::ConceptToSkosTransform.transform_document(concepts,
-                                                                      transform_options)
-        File.write(File.join(output_dir, "#{name}.jsonld"), vocab.to_jsonld)
+        require "glossarist/transforms/concept_to_gloss_transform"
+        transform = Transforms::ConceptToGlossTransform.new(nil, transform_options)
+        File.write(File.join(output_dir, "#{name}.jsonld"), transform.to_jsonld(concepts))
       end
 
       def export_turtle(concepts, name, output_dir)
-        require "glossarist/transforms/concept_to_skos_transform"
-        vocab = Transforms::ConceptToSkosTransform.transform_document(concepts,
-                                                                      transform_options)
-        File.write(File.join(output_dir, "#{name}.ttl"), vocab.to_turtle)
+        require "glossarist/transforms/concept_to_gloss_transform"
+        transform = Transforms::ConceptToGlossTransform.new(nil, transform_options)
+        File.write(File.join(output_dir, "#{name}.ttl"), transform.to_turtle(concepts))
       end
 
       def export_tbx(concepts, name, output_dir)
@@ -94,12 +92,11 @@ module Glossarist
       end
 
       def export_jsonl(concepts, name, output_dir)
-        require "glossarist/transforms/concept_to_skos_transform"
+        require "glossarist/transforms/concept_to_gloss_transform"
         File.open(File.join(output_dir, "#{name}.jsonl"), "w") do |f|
           concepts.each do |concept|
-            skos = Transforms::ConceptToSkosTransform.transform(concept,
-                                                                transform_options)
-            f.write(skos.to_jsonld)
+            transform = Transforms::ConceptToGlossTransform.new(concept, transform_options)
+            f.write(transform.to_jsonl_line)
             f.write("\n")
           end
         end

data/lib/glossarist/cli/validate_command.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "paint"
+require "table_tennis"
+
 module Glossarist
   class CLI
     class ValidateCommand
@@ -9,47 +12,135 @@ module Glossarist
       end
 
       def run
-        result = DatasetValidator.new.validate(
+        text_output = @options[:format] == "text"
+        validator = DatasetValidator.new(on_progress: text_output ? method(:print_progress) : nil)
+        result = validator.validate(
           @path,
           strict: @options[:strict],
           reference_path: @options[:reference_path],
         )
+
+        $stderr.print "\r#{' ' * 60}\r" if text_output
         report(result)
-        exit_code = result.errors.any? || (@options[:strict] && result.warnings.any?) ? 1 : 0
-        exit(exit_code) unless exit_code.zero?
+        exit(1) unless result.errors.empty? && !strict_failure?(result)
       end
 
       private
 
+      def strict_failure?(result)
+        @options[:strict] && result.warnings.any?
+      end
+
+      def print_progress(current, total)
+        pct = (current.to_f / total * 100).round
+        bar_width = 30
+        filled = (current.to_f / total * bar_width).round
+        bar = "#{'█' * filled}#{'░' * (bar_width - filled)}"
+
+        $stderr.print "\r  #{Paint['Validating', :bold]} #{bar} #{current}/#{total} (#{pct}%)"
+        $stderr.flush
+      end
+
       def report(result)
         case @options[:format]
         when "json"
-          require "json"
-          puts JSON.pretty_generate(result.to_h)
+          puts result.to_json
         when "yaml"
-          require "yaml"
-          puts YAML.dump(result.to_h)
+          puts result.to_yaml
         else
-          report_text(result)
+          print_text_output(result)
+          print_table_output(result) if result.issues.any?
         end
       end
 
-      def report_text(result)
-        if result.valid?
-          puts "Valid."
-        else
-          puts "Invalid."
-          result.errors.each { |e| puts "  ERROR: #{e}" }
+      def print_text_output(result)
+        puts
+        puts Paint["Validating #{@path}", :bold]
+        puts
+
+        if result.issues.empty?
+          puts "  #{Paint['No issues found.', :green, :bold]}"
+          return
         end
 
-        if result.warnings.any?
-          result.warnings.each do |w|
-            puts "  WARNING: #{w}"
-          end
+        print_grouped_issues(result)
+        print_summary_line(result)
+      end
+
+      def print_grouped_issues(result)
+        result.issues
+          .group_by { |i| i.location || "(dataset)" }
+          .sort_by { |loc, issues| [has_errors?(issues) ? 0 : 1, loc] }
+          .each { |location, issues| print_location_group(location, issues) }
+      end
+
+      def has_errors?(issues)
+        issues.any?(&:error?)
+      end
+
+      def print_location_group(location, issues)
+        puts "  #{Paint[location, :cyan, :bold]}"
+        issues.sort_by { |i| issue_sort_key(i) }
+          .each { |issue| print_issue(issue) }
+        puts
+      end
+
+      def issue_sort_key(issue)
+        [issue.error? ? 0 : 1, issue.code || "￿", issue.message]
+      end
+
+      def print_issue(issue)
+        color = issue.error? ? :red : :yellow
+        label = Paint[issue.error? ? "ERROR" : " WARN", color, :bold]
+        code = Paint["%-8s" % (issue.code || ""), :magenta]
+        msg_col = 21
+
+        puts "    #{label}  #{code}  #{issue.message}"
+        puts "#{' ' * msg_col}#{Paint[issue.suggestion, :green]}" if issue.suggestion
+      end
+
+      def print_summary_line(result)
+        error_count = result.issues.count(&:error?)
+        warning_count = result.issues.count(&:warning?)
+
+        status = error_count.positive? ? Paint["INVALID", :red, :bold] : Paint["VALID", :green, :bold]
+
+        details = []
+        details << Paint["#{error_count} error(s)", :red] if error_count.positive?
+        details << Paint["#{warning_count} warning(s)", :yellow] if warning_count.positive?
+
+        puts "  #{status}  #{details.join(', ')}"
+      end
+
+      def print_table_output(result)
+        rows = build_summary_rows(result)
+        return if rows.empty?
+
+        options = {
+          title: "Issues by Rule",
+          columns: %i[code severity count],
+          headers: { code: "Rule", severity: "Level", count: "Count" },
+          color_scales: { count: :gw },
+          mark: ->(row) { row[:severity] == "error" },
+          zebra: true,
+        }
+        puts
+        puts TableTennis.new(rows, options)
+      end
+
+      def build_summary_rows(result)
+        counts = Hash.new(0)
+        severities = {}
+
+        result.issues.each do |issue|
+          key = issue.code || "unknown"
+          counts[key] += 1
+          severities[key] ||= issue.severity
         end
 
-        total = result.errors.length + result.warnings.length
-        puts "#{total} issue(s) found."
+        counts.sort_by { |_, c| -c }.map do |code, count|
+          { code: code, severity: severities[code], count: count }
+        end
       end
     end
   end

data/lib/glossarist/cli.rb

@@ -9,6 +9,7 @@ module Glossarist
     autoload :ValidateCommand, "#{__dir__}/cli/validate_command"
     autoload :ImportCommand,   "#{__dir__}/cli/import_command"
     autoload :ExportCommand,   "#{__dir__}/cli/export_command"
+    autoload :CompareCommand,  "#{__dir__}/cli/compare_command"
     desc "generate_latex", "Convert Concepts to Latex format"
 
     option :concepts_path, aliases: :p, required: true,
@@ -120,6 +121,23 @@ module Glossarist
       CLI::ExportCommand.new(path, options).run
     end
 
+    desc "compare NEW_PATH OLD_PATH", "Compare two concept datasets"
+    option :format, type: :string, default: "text",
+                    enum: %w[text json yaml],
+                    desc: "Output format"
+    option :report, type: :string,
+                    desc: "Write report to file"
+    option :no_diffs, type: :boolean, default: false,
+                      desc: "Skip per-concept diff computation"
+    def compare(new_path, old_path)
+      CLI::CompareCommand.new(new_path, old_path, options).run
+    end
+
+    desc "version", "Show Glossarist version"
+    def version
+      puts Glossarist::VERSION
+    end
+
     def method_missing(*args)
       warn "No method found named: #{args[0]}"
       warn "Run with `--help` or `-h` to see available options"

data/lib/glossarist/collections/bibliography_collection.rb

@@ -39,9 +39,11 @@ module Glossarist
       def populate_bibliographies(concepts)
         concepts.each do |concept|
           concept.default_lang.sources.each do |source|
-            next if source.origin.text.nil?
+            next if source.origin.ref.nil?
+            ref_text = source.origin.ref.source
+            next if ref_text.nil?
 
-            fetch(source.origin.text)
+            fetch(ref_text)
           end
         end
       end

data/lib/glossarist/collections/localization_collection.rb

@@ -7,6 +7,8 @@ module Glossarist
 
       index_by :language_code
 
+      alias :length :size
+
       def [](lang_code)
         find_by(:language_code, lang_code.to_s)
       end