glossarist 2.5.0 → 2.5.2

data/lib/glossarist/gcr_validator.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Glossarist
+  class GcrValidator
+    def validate(zip_path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      result = ValidationResult.new
+
+      unless File.exist?(zip_path)
+        result.add_error("File not found: #{zip_path}")
+        return result
+      end
+
+      begin
+        Zip::File.open(zip_path) do |zf|
+          unless zf.find_entry("metadata.yaml")
+            result.add_error("Missing metadata.yaml")
+            return result
+          end
+
+          metadata = GcrMetadata.from_yaml(
+            zf.find_entry("metadata.yaml").get_input_stream.read,
+          )
+          validate_metadata(metadata, result)
+
+          concept_entries = zf.entries.select do |e|
+            e.name.start_with?("concepts/") && e.name.end_with?(".yaml")
+          end
+          if concept_entries.empty?
+            result.add_error("No concept files found in concepts/")
+          end
+
+          concept_entries.each do |entry|
+            validate_concept_entry(entry, metadata, result)
+          end
+        end
+      rescue StandardError => e
+        result.add_error("Failed to read ZIP: #{e.message}")
+      end
+
+      result
+    end
+
+    private
+
+    def validate_metadata(metadata, result)
+      unless metadata&.concept_count
+        result.add_error("metadata.yaml missing required fields (concept_count)")
+      end
+
+      unless metadata&.shortname
+        result.add_error("metadata.yaml missing shortname")
+      end
+
+      unless metadata&.version
+        result.add_error("metadata.yaml missing version")
+      end
+    end
+
+    def validate_concept_entry(entry, metadata, result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      raw = entry.get_input_stream.read
+      doc = ConceptDocument.from_yamls(raw)
+    rescue Psych::SyntaxError => e
+      result.add_error("#{entry.name}: invalid YAML at line #{e.line}: #{e.message}")
+    rescue StandardError => e
+      result.add_error("#{entry.name}: parse error: #{e.message}")
+    else
+      concept = doc.concept
+      unless concept&.data&.id
+        result.add_error("#{entry.name}: document 0 missing data.identifier")
+      end
+
+      localizations = doc.localizations
+      if localizations.empty?
+        result.add_error("#{entry.name}: expected at least 1 localization document")
+      else
+        localizations.each_with_index do |l10n, idx|
+          unless l10n&.language_code
+            result.add_error("#{entry.name}: document #{idx + 1} missing data.language_code")
+          end
+        end
+      end
+
+      validate_concept_uri(entry, concept, metadata, result)
+    end
+
+    def validate_concept_uri(entry, concept, metadata, result) # rubocop:disable Metrics/CyclomaticComplexity
+      concept_uri = concept&.data&.uri
+      template = metadata&.concept_uri_template
+      uri_prefix = metadata&.uri_prefix
+
+      if concept_uri.nil? && template.nil? && uri_prefix.nil?
+        result.add_warning("#{entry.name}: no concept URI (data.uri) and no concept_uri_template or uri_prefix in metadata")
+      end
+    end
+  end
+end

data/lib/glossarist/locality.rb

@@ -13,7 +13,7 @@ module Glossarist
     # @return [String]
     attribute :reference_to, :string
 
-    yaml do
+    key_value do
       map :type, to: :type
       map :reference_from, to: :reference_from
       map :reference_to, to: :reference_to

data/lib/glossarist/localized_concept.rb

@@ -4,11 +4,22 @@ module Glossarist
     attribute :review_type, :string
     attribute :entry_status, :string
 
-    yaml do
+    key_value do
       map :classification, to: :classification
       map %i[review_type reviewType], to: :review_type
     end
 
+    NIL_COLLECTION_KEYS = %w[definition examples notes].freeze
+
+    def self.of_yaml(hash, options = {})
+      if hash.is_a?(Hash) && (data = hash["data"]).is_a?(Hash)
+        NIL_COLLECTION_KEYS.each do |key|
+          data[key] = [] if data.key?(key) && data[key].nil?
+        end
+      end
+      super
+    end
+
     alias_method :status=, :entry_status=
 
     def language_code

data/lib/glossarist/managed_concept.rb

@@ -21,7 +21,7 @@ module Glossarist
 
     attribute :uuid, :string
 
-    yaml do
+    key_value do
       map :data, to: :data
       map :id, with: { to: :identifier_to_yaml, from: :identifier_from_yaml }
       map :identifier,

data/lib/glossarist/managed_concept_data.rb

@@ -3,6 +3,7 @@ module Glossarist
     include Glossarist::Utilities::CommonFunctions
 
     attribute :id, :string
+    attribute :uri, :string
     attribute :localized_concepts, :hash
     attribute :groups, :string, collection: true
     attribute :sources, ConceptSource, collection: true
@@ -10,9 +11,10 @@ module Glossarist
               collection: Collections::LocalizationCollection,
               initialize_empty: true
 
-    yaml do
+    key_value do
       map %i[id identifier], to: :id,
                              with: { to: :id_to_yaml, from: :id_from_yaml }
+      map :uri, to: :uri
       map %i[localized_concepts localizedConcepts], to: :localized_concepts
       map :groups, to: :groups
       map :sources, to: :sources
@@ -32,7 +34,8 @@ module Glossarist
     def localizations_from_yaml(model, value)
       value.each do |localized_concept_hash|
         localized_concept = Glossarist::LocalizedConcept.of_yaml(localized_concept_hash)
-        model.localizations.store(localized_concept.language_code, localized_concept)
+        model.localizations.store(localized_concept.language_code,
+                                  localized_concept)
       end
     end
 

data/lib/glossarist/non_verb_rep.rb

@@ -5,7 +5,7 @@ module Glossarist
     attribute :formula, :string
     attribute :sources, ConceptSource, collection: true
 
-    yaml do
+    key_value do
       map :image, to: :image
       map :table, to: :table
       map :formula, to: :formula

data/lib/glossarist/reference_extractor.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Glossarist
+  class ReferenceExtractor
+    @identifier_resolvers = []
+    @patterns = []
+
+    IdentifierResolver = Struct.new(:prefix, :resolver, keyword_init: true)
+    Pattern = Struct.new(:name, :regex, :resolver, keyword_init: true)
+
+    class << self
+      def register_identifier_resolver(prefix, &resolver)
+        @identifier_resolvers << IdentifierResolver.new(prefix: prefix,
+                                                        resolver: resolver)
+      end
+
+      def register_pattern(name:, regex:, &resolver)
+        @patterns << Pattern.new(name: name, regex: regex, resolver: resolver)
+      end
+
+      def patterns
+        @patterns.dup
+      end
+
+      def identifier_resolvers
+        @identifier_resolvers.dup
+      end
+    end
+
+    def extract_from_text(text)
+      return [] unless text.is_a?(String)
+
+      refs = []
+      self.class.patterns.each do |pattern|
+        text.scan(pattern.regex).each do |captures|
+          captures = [captures] unless captures.is_a?(Array)
+          ref = pattern.resolver.call(self, *captures)
+          refs << ref if ref
+        end
+      end
+      deduplicate(refs)
+    end
+
+    def extract_from_localized(lc_hash)
+      gather_texts(lc_hash).flat_map { |t| extract_from_text(t) }
+    end
+
+    def extract_from_concept_hash(concept_hash)
+      LANG_CODES.flat_map do |lang|
+        next [] unless concept_hash[lang].is_a?(Hash)
+
+        extract_from_localized(concept_hash[lang])
+      end
+    end
+
+    def extract_from_managed_concept(concept)
+      concept.localizations.flat_map do |l10n|
+        extract_from_localized_concept(l10n)
+      end
+    end
+
+    def extract_from_localized_concept(l10n)
+      texts = []
+      l10n.data.definition&.each { |d| texts << d.content if d.content }
+      l10n.data.notes&.each { |n| texts << n.content if n.content }
+      l10n.data.examples&.each { |e| texts << e.content if e.content }
+      texts.flat_map { |t| extract_from_text(t) }
+    end
+
+    # Unified concept mention dispatcher.
+    # Content is the text inside {{...}}.
+    def resolve_mention(content)
+      content = content.strip
+
+      if content.include?(",")
+        parts = content.split(",", 2)
+        display = parts[0].strip
+        identifier = parts[1].strip
+        resolve_by_identifier(identifier, display)
+      else
+        resolve_by_identifier(content, nil)
+      end
+    end
+
+    def resolve_by_identifier(identifier, display)
+      # Check registered identifier resolvers (built-in + custom)
+      self.class.identifier_resolvers.each do |ir|
+        next unless identifier.start_with?(ir.prefix)
+
+        return ir.resolver.call(self, identifier, display)
+      end
+
+      case identifier
+      when /\A\d[\d.-]*\z/
+        resolve_local(display || identifier, identifier)
+      else
+        resolve_designation(identifier, display)
+      end
+    end
+
+    def resolve_local(term, concept_id)
+      ConceptReference.new(
+        term: term.strip,
+        concept_id: concept_id.strip,
+        source: nil,
+        ref_type: "local",
+      )
+    end
+
+    def resolve_designation(text, display)
+      ConceptReference.new(
+        term: display || text,
+        concept_id: nil,
+        source: nil,
+        ref_type: "designation",
+      )
+    end
+
+    def resolve_iec_urn(urn, display)
+      concept_id = extract_iec_concept_id(urn)
+
+      ConceptReference.new(
+        term: display || "",
+        concept_id: concept_id,
+        source: "urn:iec:std:iec:60050",
+        ref_type: "urn",
+      )
+    end
+
+    def resolve_iso_urn(urn, display)
+      if (m = urn.match(/\Aurn:iso:std:iso:(\d+)(?::(.*))?\z/))
+        term_id = extract_term_id_from_urn_tail(m[2])
+        ConceptReference.new(
+          term: display || "",
+          concept_id: term_id,
+          source: "urn:iso:std:iso:#{m[1]}",
+          ref_type: "urn",
+        )
+      end
+    end
+
+    def resolve_generic_urn(urn, display)
+      ConceptReference.new(
+        term: display || "",
+        concept_id: nil,
+        source: urn,
+        ref_type: "urn",
+      )
+    end
+
+    LANG_CODES = Glossarist::LANG_CODES
+
+    private
+
+    def gather_texts(lc_hash)
+      texts = extract_text_fields(lc_hash["definition"])
+      texts << lc_hash["definition"].to_s if lc_hash["definition"].is_a?(String)
+      texts.concat(extract_text_fields(lc_hash["notes"]))
+      texts.concat(extract_text_fields(lc_hash["examples"]))
+      texts
+    end
+
+    def extract_text_fields(items)
+      Array(items).filter_map do |item|
+        item.is_a?(Hash) ? item["content"]&.to_s : item.to_s
+      end
+    end
+
+    def deduplicate(refs)
+      seen = Set.new
+      refs.select do |ref|
+        key = if ref.concept_id
+                [ref.source,
+                 ref.concept_id]
+              else
+                [ref.source, ref.concept_id,
+                 ref.term]
+              end
+        seen.add?(key)
+      end
+    end
+
+    def extract_term_id_from_urn_tail(tail)
+      return "" unless tail
+
+      if (m = tail.match(/term:([\d.,]+)/))
+        m[1].split(",").first
+      elsif (m = tail.match(/sec:([\d.]+)/))
+        m[1]
+      else
+        tail
+      end
+    end
+
+    def extract_iec_concept_id(urn)
+      if (m = urn.match(/::#con-([\d-]+)/))
+        m[1]
+      else
+        segments = urn.split(":")
+        code_part = segments.find { |s| s.start_with?("60050-") }
+        return "" unless code_part
+
+        code_part.delete_prefix("60050-").sub(/-\d{4}-\d{2}\z/, "")
+      end
+    end
+
+    # Unified concept mention pattern: {{...}}
+    register_pattern(
+      name: :concept_mention,
+      regex: /\{\{([^}]+)\}\}/,
+    ) { |ext, content| ext.resolve_mention(content) }
+
+    register_identifier_resolver("urn:iec:std:iec:60050") do |ext, identifier, display|
+      ext.resolve_iec_urn(identifier, display)
+    end
+
+    register_identifier_resolver("urn:iso:std:iso:") do |ext, identifier, display|
+      ext.resolve_iso_urn(identifier, display)
+    end
+
+    register_identifier_resolver("urn:") do |ext, identifier, display|
+      ext.resolve_generic_urn(identifier, display)
+    end
+  end
+end

data/lib/glossarist/reference_resolver.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ReferenceResolver
+    def initialize
+      @local_adapter = nil
+      @package_adapters = []
+      @route_adapter = ResolutionAdapter::Route.new
+      @remote_adapters = []
+    end
+
+    def register_self(concepts)
+      @local_adapter = ResolutionAdapter::Local.new(concepts)
+    end
+
+    def register_package(package_or_concepts, uri_prefix: nil)
+      concepts = extract_concepts(package_or_concepts)
+      prefix = uri_prefix || infer_uri_prefix(package_or_concepts)
+      raise ArgumentError, "uri_prefix required" unless prefix
+
+      @package_adapters << ResolutionAdapter::Package.new(concepts,
+                                                          uri_prefix: prefix)
+    end
+
+    def add_route(from:, to:)
+      @route_adapter.add(from: from, to: to)
+    end
+
+    def register_remote(uri_prefix:, endpoint:)
+      @remote_adapters << ResolutionAdapter::Remote.new(uri_prefix: uri_prefix,
+                                                        endpoint: endpoint)
+    end
+
+    def resolve(reference)
+      if reference.local?
+        return @local_adapter&.resolve(reference)
+      end
+
+      routed_ref = apply_routes(reference)
+
+      @package_adapters.each do |adapter|
+        result = adapter.resolve(routed_ref)
+        return result if result
+      end
+
+      @remote_adapters.each do |adapter|
+        result = adapter.resolve(routed_ref)
+        return result if result
+      end
+
+      nil
+    end
+
+    def resolve_all(concept, extractor: nil)
+      extractor ||= ReferenceExtractor.new
+      refs = extract_refs(concept, extractor)
+      refs.map { |ref| [ref, resolve(ref)] }
+    end
+
+    def validate_all(package_or_concepts, extractor: nil, mode: :multi)
+      concepts = extract_concepts(package_or_concepts)
+      extractor ||= ReferenceExtractor.new
+      result = ValidationResult.new
+
+      concepts.each do |concept|
+        refs = extract_refs(concept, extractor)
+        termid = extract_termid(concept)
+
+        refs.each do |ref|
+          resolved = resolve(ref)
+          if resolved.nil?
+            scope = ref.local? ? "intra-set" : "inter-set (#{ref.source})"
+            result.add_warning("#{termid}: Unresolvable #{scope} reference: #{ref.term} -> #{ref.concept_id}")
+          end
+        end
+
+        if mode == :single && !@local_adapter
+          external_refs = refs.select(&:external?)
+          if external_refs.any?
+            result.add_warning("#{termid}: #{external_refs.size} external reference(s) not checked in single mode")
+          end
+        end
+      end
+
+      result
+    end
+
+    def load_collection(collection_dir)
+      config_path = File.join(collection_dir, "collection.yaml")
+      if File.exist?(config_path)
+        load_collection_config(config_path, collection_dir)
+      else
+        load_gcr_directory(collection_dir)
+      end
+    end
+
+    def registered_datasets
+      @package_adapters.map(&:uri_prefix)
+    end
+
+    private
+
+    def apply_routes(reference)
+      routed = @route_adapter.resolve(reference)
+      routed || reference
+    end
+
+    def extract_concepts(package_or_concepts)
+      case package_or_concepts
+      when GcrPackage then package_or_concepts.concepts
+      when Array then package_or_concepts
+      when Hash then [package_or_concepts]
+      else raise ArgumentError, "Expected GcrPackage, Array, or Hash"
+      end
+    end
+
+    def infer_uri_prefix(package_or_concepts)
+      case package_or_concepts
+      when GcrPackage then package_or_concepts.metadata&.dig("uri_prefix")
+      end
+    end
+
+    def load_collection_config(config_path, collection_dir)
+      config = CollectionConfig.from_file(config_path)
+
+      config.packages.each do |pkg|
+        gcr_path = File.join(collection_dir, pkg["file"])
+        next unless File.exist?(gcr_path)
+
+        gcr = GcrPackage.load(gcr_path)
+        prefix = pkg["uri_prefix"] || gcr.metadata&.dig("uri_prefix")
+        register_package(gcr, uri_prefix: prefix)
+      end
+
+      config.routes.each do |route|
+        add_route(from: route["from"], to: route["to"])
+      end
+
+      config.remotes.each do |remote|
+        register_remote(uri_prefix: remote["uri_prefix"],
+                        endpoint: remote["endpoint"])
+      end
+    end
+
+    def load_gcr_directory(dir)
+      Dir.glob(File.join(dir, "*.gcr")).each do |gcr_path|
+        pkg = GcrPackage.load(gcr_path)
+        prefix = pkg.metadata&.dig("uri_prefix")
+        register_package(pkg, uri_prefix: prefix)
+      end
+    end
+
+    def extract_refs(concept, extractor)
+      if concept.is_a?(ManagedConcept)
+        extractor.extract_from_managed_concept(concept)
+      else
+        extractor.extract_from_concept_hash(concept)
+      end
+    end
+
+    def extract_termid(concept)
+      if concept.is_a?(ManagedConcept)
+        concept.data.id&.to_s
+      else
+        (concept["termid"] || concept["id"])&.to_s
+      end
+    end
+  end
+end

data/lib/glossarist/register_data.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class RegisterData < Lutaml::Model::Serializable
+    attribute :data, :hash, default: -> { {} }
+
+    key_value do
+      map nil, to: :data, with: { from: :data_from, to: :data_to }
+    end
+
+    def self.from_file(path)
+      from_yaml(File.read(path))
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def [](key)
+      data[key]
+    end
+
+    def dig(*keys)
+      data.dig(*keys)
+    end
+
+    def to_h
+      data
+    end
+
+    def data_from(model, value)
+      model.data = value
+    end
+
+    def data_to(model, doc)
+      model.data.each do |key, value|
+        doc[key] = value
+      end
+    end
+  end
+end

data/lib/glossarist/related_concept.rb

@@ -7,7 +7,7 @@ module Glossarist
               values: Glossarist::GlossaryDefinition::RELATED_CONCEPT_TYPES
     attribute :ref, Citation
 
-    yaml do
+    key_value do
       map :content, to: :content
       map :type, to: :type
       map :ref, with: { from: :ref_from_yaml, to: :ref_to_yaml }

data/lib/glossarist/resolution_adapter/local.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ResolutionAdapter
+    class Local < ResolutionAdapter
+      attr_reader :index, :concepts
+
+      def initialize(concepts)
+        super()
+        @concepts = concepts
+        @index = {}
+        build_index
+      end
+
+      def resolve(reference)
+        case reference.ref_type
+        when "local"
+          resolve_by_id(reference.concept_id)
+        when "designation"
+          resolve_by_designation(reference.term)
+        else
+          resolve_by_id(reference.concept_id) if reference.concept_id
+        end
+      end
+
+      def resolve_by_id(concept_id)
+        @index[concept_id]
+      end
+
+      def resolve_by_designation(term)
+        return nil unless term
+
+        downcased = term.downcase
+        concepts.find do |concept|
+          designations_for(concept).any? { |d| d&.downcase == downcased }
+        end
+      end
+
+      private
+
+      def designations_for(concept)
+        if concept.is_a?(ManagedConcept)
+          concept.localizations.flat_map do |l10n|
+            l10n.data.terms.filter_map do |t|
+              t.respond_to?(:designation) ? t.designation : nil
+            end
+          end
+        else
+          concept.each_value.flat_map do |lang_block|
+            next [] unless lang_block.is_a?(Hash) && lang_block.key?("terms")
+
+            Array(lang_block["terms"]).filter_map { |t| t["designation"] }
+          end
+        end
+      end
+
+      def build_index
+        concepts.each do |concept|
+          termid = extract_termid(concept)
+          @index[termid] = concept if termid
+        end
+      end
+
+      def extract_termid(concept)
+        if concept.is_a?(ManagedConcept)
+          concept.data.id&.to_s
+        else
+          (concept["termid"] || concept["id"])&.to_s
+        end
+      end
+    end
+  end
+end