glossarist 2.4.0 → 2.5.1

data/lib/glossarist/gcr_package.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "zip"
+require "fileutils"
+
+module Glossarist
+  class GcrPackage
+    attr_reader :zip_path, :metadata, :concepts
+
+    def initialize(zip_path)
+      @zip_path = zip_path
+      @metadata = nil
+      @concepts = []
+    end
+
+    def self.create(concepts:, metadata:, output_path:, register_data: nil)
+      FileUtils.mkdir_p(File.dirname(output_path))
+      package = new(output_path)
+      package.send(:write, concepts, metadata, register_data)
+      package
+    end
+
+    def self.load(zip_path)
+      package = new(zip_path)
+      package.send(:read)
+      package
+    end
+
+    def self.create_from_directory(dir, output:, shortname:, version:, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+                                  title: nil, description: nil, owner: nil,
+                                  tags: [], register_yaml: nil,
+                                  uri_prefix: nil, concept_uri_template: nil,
+                                  streaming: false)
+      dir = File.expand_path(dir)
+
+      if streaming
+        create_streaming(dir, output: output, shortname: shortname, version: version,
+                              title: title, description: description, owner: owner,
+                              tags: tags, register_yaml: register_yaml,
+                              uri_prefix: uri_prefix,
+                              concept_uri_template: concept_uri_template)
+      else
+        create_batch(dir, output: output, shortname: shortname, version: version,
+                          title: title, description: description, owner: owner,
+                          tags: tags, register_yaml: register_yaml,
+                          uri_prefix: uri_prefix,
+                          concept_uri_template: concept_uri_template)
+      end
+    end
+
+    def validate
+      GcrValidator.new.validate(@zip_path)
+    end
+
+    private
+
+    def write(concepts, metadata, register_data)
+      Zip::File.open(@zip_path, create: true) do |zf|
+        zf.get_output_stream("metadata.yaml") do |f|
+          f.write(metadata.to_yaml)
+        end
+
+        if register_data
+          zf.get_output_stream("register.yaml") do |f|
+            f.write(register_data.to_yaml)
+          end
+        end
+
+        concepts.each do |mc|
+          write_concept(zf, mc)
+        end
+      end
+    end
+
+    def write_concept(zip_file, concept)
+      termid = concept.data.id.to_s
+      doc = ConceptDocument.from_managed_concept(concept)
+      zip_file.get_output_stream("concepts/#{termid}.yaml") do |f|
+        f.write(doc.to_yamls)
+      end
+    end
+
+    def read
+      @concepts = []
+
+      Zip::File.open(@zip_path) do |zf|
+        if (entry = zf.find_entry("metadata.yaml"))
+          @metadata = GcrMetadata.from_yaml(entry.get_input_stream.read)
+        end
+
+        zf.entries.each do |entry|
+          next unless entry.name.start_with?("concepts/") && entry.name.end_with?(".yaml")
+
+          raw = entry.get_input_stream.read
+          doc = ConceptDocument.from_yamls(raw)
+          @concepts << doc.to_managed_concept
+        end
+      end
+    end
+
+    class << self
+      private
+
+      def create_batch(dir, output:, shortname:, version:, **opts)
+        concepts = ConceptCollector.collect(dir)
+        if concepts.empty?
+          raise ArgumentError,
+                "No concept files found in #{dir}"
+        end
+
+        enricher = ConceptEnricher.new
+        enricher.inject_references(concepts)
+        if opts[:concept_uri_template]
+          enricher.apply_uri_template(concepts,
+                                      opts[:concept_uri_template])
+        end
+
+        register_data = load_register_data(opts[:register_yaml], dir)
+        metadata = build_metadata(concepts, shortname: shortname, version: version,
+                                            register_data: register_data, **opts)
+
+        create(
+          concepts: concepts,
+          metadata: metadata,
+          register_data: register_data,
+          output_path: File.expand_path(output),
+        )
+      end
+
+      def create_streaming(dir, output:, shortname:, version:, **opts) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
+        enricher = ConceptEnricher.new
+        output_path = File.expand_path(output)
+        FileUtils.mkdir_p(File.dirname(output_path))
+
+        register_data = load_register_data(opts[:register_yaml], dir)
+        concept_count = 0
+        languages = Set.new
+
+        Zip::OutputStream.open(output_path) do |zos|
+          if register_data
+            zos.put_next_entry("register.yaml")
+            zos.write(register_data.to_yaml)
+          end
+
+          ConceptCollector.each_concept(dir) do |mc|
+            enricher.inject_references([mc])
+            if opts[:concept_uri_template]
+              enricher.apply_uri_template([mc],
+                                          opts[:concept_uri_template])
+            end
+
+            mc.localizations.each do |l10n|
+              languages << l10n.language_code if l10n.language_code
+            end
+            concept_count += 1
+
+            termid = mc.data.id.to_s
+            doc = ConceptDocument.from_managed_concept(mc)
+            zos.put_next_entry("concepts/#{termid}.yaml")
+            zos.write(doc.to_yamls)
+          end
+
+          if concept_count.zero?
+            raise ArgumentError,
+                  "No concept files found in #{dir}"
+          end
+
+          metadata = build_streaming_metadata(concept_count, languages,
+                                              shortname: shortname, version: version,
+                                              register_data: register_data, **opts)
+          zos.put_next_entry("metadata.yaml")
+          zos.write(metadata.to_yaml)
+        end
+
+        new(output_path)
+      end
+
+      def build_streaming_metadata(concept_count, languages, shortname:, version:, # rubocop:disable Metrics/ParameterLists
+                                   register_data: nil, **opts)
+        GcrMetadata.new(
+          shortname: shortname,
+          version: version,
+          title: opts[:title],
+          description: opts[:description],
+          owner: opts[:owner],
+          tags: opts[:tags] || [],
+          concept_count: concept_count,
+          languages: languages.sort,
+          created_at: Time.now.utc.iso8601,
+          glossarist_version: Glossarist::VERSION,
+          schema_version: register_data&.dig("schema_version") || SchemaMigration::CURRENT_SCHEMA_VERSION,
+          uri_prefix: opts[:uri_prefix],
+          concept_uri_template: opts[:concept_uri_template],
+        )
+      end
+
+      def build_metadata(concepts, shortname:, version:, register_data: nil,
+**opts)
+        GcrMetadata.from_concepts(
+          concepts,
+          register_data: register_data,
+          options: {
+            shortname: shortname,
+            version: version,
+            title: opts[:title],
+            description: opts[:description],
+            owner: opts[:owner],
+            tags: opts[:tags],
+            uri_prefix: opts[:uri_prefix],
+            concept_uri_template: opts[:concept_uri_template],
+          },
+        )
+      end
+
+      def load_register_data(register_yaml_path, dir)
+        path = register_yaml_path || File.join(dir, "register.yaml")
+        return nil unless File.exist?(path)
+
+        RegisterData.from_file(path)
+      end
+    end
+  end
+end

data/lib/glossarist/gcr_statistics.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class GcrStatistics < Lutaml::Model::Serializable
+    attribute :total_concepts, :integer
+    attribute :languages, :string, collection: true
+    attribute :concepts_by_status, :hash
+    attribute :concepts_with_definitions, :integer
+    attribute :concepts_with_sources, :integer
+
+    key_value do
+      map :total_concepts, to: :total_concepts
+      map :languages, to: :languages
+      map :concepts_by_status, to: :concepts_by_status
+      map :concepts_with_definitions, to: :concepts_with_definitions
+      map :concepts_with_sources, to: :concepts_with_sources
+    end
+
+    def self.from_concepts(concepts)
+      l10ns = concepts.flat_map { |c| c.localizations.to_a }
+
+      new(
+        total_concepts: concepts.length,
+        languages: l10ns.map(&:language_code).compact.sort.uniq,
+        concepts_by_status: l10ns.map(&:entry_status).compact.tally,
+        concepts_with_definitions: count_with(l10ns, :definition),
+        concepts_with_sources: count_with(l10ns, :sources),
+      )
+    end
+
+    def self.count_with(l10ns, attr)
+      l10ns.count { |l| l.data.send(attr)&.any? }
+    end
+  end
+end

data/lib/glossarist/gcr_validator.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "zip"
+
+module Glossarist
+  class GcrValidator
+    def validate(zip_path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      result = ValidationResult.new
+
+      unless File.exist?(zip_path)
+        result.add_error("File not found: #{zip_path}")
+        return result
+      end
+
+      begin
+        Zip::File.open(zip_path) do |zf|
+          unless zf.find_entry("metadata.yaml")
+            result.add_error("Missing metadata.yaml")
+            return result
+          end
+
+          metadata = GcrMetadata.from_yaml(
+            zf.find_entry("metadata.yaml").get_input_stream.read,
+          )
+          validate_metadata(metadata, result)
+
+          concept_entries = zf.entries.select do |e|
+            e.name.start_with?("concepts/") && e.name.end_with?(".yaml")
+          end
+          if concept_entries.empty?
+            result.add_error("No concept files found in concepts/")
+          end
+
+          concept_entries.each do |entry|
+            validate_concept_entry(entry, metadata, result)
+          end
+        end
+      rescue StandardError => e
+        result.add_error("Failed to read ZIP: #{e.message}")
+      end
+
+      result
+    end
+
+    private
+
+    def validate_metadata(metadata, result)
+      unless metadata&.concept_count
+        result.add_error("metadata.yaml missing required fields (concept_count)")
+      end
+
+      unless metadata&.shortname
+        result.add_error("metadata.yaml missing shortname")
+      end
+
+      unless metadata&.version
+        result.add_error("metadata.yaml missing version")
+      end
+    end
+
+    def validate_concept_entry(entry, metadata, result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      raw = entry.get_input_stream.read
+      doc = ConceptDocument.from_yamls(raw)
+    rescue Psych::SyntaxError => e
+      result.add_error("#{entry.name}: invalid YAML at line #{e.line}: #{e.message}")
+    rescue StandardError => e
+      result.add_error("#{entry.name}: parse error: #{e.message}")
+    else
+      concept = doc.concept
+      unless concept&.data&.id
+        result.add_error("#{entry.name}: document 0 missing data.identifier")
+      end
+
+      localizations = doc.localizations
+      if localizations.empty?
+        result.add_error("#{entry.name}: expected at least 1 localization document")
+      else
+        localizations.each_with_index do |l10n, idx|
+          unless l10n&.language_code
+            result.add_error("#{entry.name}: document #{idx + 1} missing data.language_code")
+          end
+        end
+      end
+
+      validate_concept_uri(entry, concept, metadata, result)
+    end
+
+    def validate_concept_uri(entry, concept, metadata, result) # rubocop:disable Metrics/CyclomaticComplexity
+      concept_uri = concept&.data&.uri
+      template = metadata&.concept_uri_template
+      uri_prefix = metadata&.uri_prefix
+
+      if concept_uri.nil? && template.nil? && uri_prefix.nil?
+        result.add_warning("#{entry.name}: no concept URI (data.uri) and no concept_uri_template or uri_prefix in metadata")
+      end
+    end
+  end
+end

data/lib/glossarist/locality.rb

@@ -13,7 +13,7 @@ module Glossarist
     # @return [String]
     attribute :reference_to, :string
 
-    yaml do
+    key_value do
       map :type, to: :type
       map :reference_from, to: :reference_from
       map :reference_to, to: :reference_to

data/lib/glossarist/localized_concept.rb

@@ -4,11 +4,22 @@ module Glossarist
     attribute :review_type, :string
     attribute :entry_status, :string
 
-    yaml do
+    key_value do
       map :classification, to: :classification
       map %i[review_type reviewType], to: :review_type
     end
 
+    NIL_COLLECTION_KEYS = %w[definition examples notes].freeze
+
+    def self.of_yaml(hash, options = {})
+      if hash.is_a?(Hash) && (data = hash["data"]).is_a?(Hash)
+        NIL_COLLECTION_KEYS.each do |key|
+          data[key] = [] if data.key?(key) && data[key].nil?
+        end
+      end
+      super
+    end
+
     alias_method :status=, :entry_status=
 
     def language_code

data/lib/glossarist/managed_concept.rb

@@ -21,7 +21,7 @@ module Glossarist
 
     attribute :uuid, :string
 
-    yaml do
+    key_value do
       map :data, to: :data
       map :id, with: { to: :identifier_to_yaml, from: :identifier_from_yaml }
       map :identifier,

data/lib/glossarist/managed_concept_data.rb

@@ -3,14 +3,18 @@ module Glossarist
     include Glossarist::Utilities::CommonFunctions
 
     attribute :id, :string
+    attribute :uri, :string
     attribute :localized_concepts, :hash
     attribute :groups, :string, collection: true
     attribute :sources, ConceptSource, collection: true
-    attribute :localizations, :hash, collection: true, default: -> { {} }
+    attribute :localizations, LocalizedConcept,
+              collection: Collections::LocalizationCollection,
+              initialize_empty: true
 
-    yaml do
+    key_value do
       map %i[id identifier], to: :id,
                              with: { to: :id_to_yaml, from: :id_from_yaml }
+      map :uri, to: :uri
       map %i[localized_concepts localizedConcepts], to: :localized_concepts
       map :groups, to: :groups
       map :sources, to: :sources
@@ -28,11 +32,10 @@ module Glossarist
     end
 
     def localizations_from_yaml(model, value)
-      model.localizations ||= {}
-
       value.each do |localized_concept_hash|
         localized_concept = Glossarist::LocalizedConcept.of_yaml(localized_concept_hash)
-        model.localizations[localized_concept.language_code] = localized_concept
+        model.localizations.store(localized_concept.language_code,
+                                  localized_concept)
       end
     end
 

data/lib/glossarist/non_verb_rep.rb

@@ -5,7 +5,7 @@ module Glossarist
     attribute :formula, :string
     attribute :sources, ConceptSource, collection: true
 
-    yaml do
+    key_value do
       map :image, to: :image
       map :table, to: :table
       map :formula, to: :formula

data/lib/glossarist/reference_extractor.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Glossarist
+  class ReferenceExtractor
+    @identifier_resolvers = []
+    @patterns = []
+
+    IdentifierResolver = Struct.new(:prefix, :resolver, keyword_init: true)
+    Pattern = Struct.new(:name, :regex, :resolver, keyword_init: true)
+
+    class << self
+      def register_identifier_resolver(prefix, &resolver)
+        @identifier_resolvers << IdentifierResolver.new(prefix: prefix,
+                                                        resolver: resolver)
+      end
+
+      def register_pattern(name:, regex:, &resolver)
+        @patterns << Pattern.new(name: name, regex: regex, resolver: resolver)
+      end
+
+      def patterns
+        @patterns.dup
+      end
+
+      def identifier_resolvers
+        @identifier_resolvers.dup
+      end
+    end
+
+    def extract_from_text(text)
+      return [] unless text.is_a?(String)
+
+      refs = []
+      self.class.patterns.each do |pattern|
+        text.scan(pattern.regex).each do |captures|
+          captures = [captures] unless captures.is_a?(Array)
+          ref = pattern.resolver.call(self, *captures)
+          refs << ref if ref
+        end
+      end
+      deduplicate(refs)
+    end
+
+    def extract_from_localized(lc_hash)
+      gather_texts(lc_hash).flat_map { |t| extract_from_text(t) }
+    end
+
+    def extract_from_concept_hash(concept_hash)
+      LANG_CODES.flat_map do |lang|
+        next [] unless concept_hash[lang].is_a?(Hash)
+
+        extract_from_localized(concept_hash[lang])
+      end
+    end
+
+    def extract_from_managed_concept(concept)
+      concept.localizations.flat_map do |l10n|
+        extract_from_localized_concept(l10n)
+      end
+    end
+
+    def extract_from_localized_concept(l10n)
+      texts = []
+      l10n.data.definition&.each { |d| texts << d.content if d.content }
+      l10n.data.notes&.each { |n| texts << n.content if n.content }
+      l10n.data.examples&.each { |e| texts << e.content if e.content }
+      texts.flat_map { |t| extract_from_text(t) }
+    end
+
+    # Unified concept mention dispatcher.
+    # Content is the text inside {{...}}.
+    def resolve_mention(content)
+      content = content.strip
+
+      if content.include?(",")
+        parts = content.split(",", 2)
+        display = parts[0].strip
+        identifier = parts[1].strip
+        resolve_by_identifier(identifier, display)
+      else
+        resolve_by_identifier(content, nil)
+      end
+    end
+
+    def resolve_by_identifier(identifier, display)
+      # Check registered identifier resolvers (built-in + custom)
+      self.class.identifier_resolvers.each do |ir|
+        next unless identifier.start_with?(ir.prefix)
+
+        return ir.resolver.call(self, identifier, display)
+      end
+
+      case identifier
+      when /\A\d[\d.-]*\z/
+        resolve_local(display || identifier, identifier)
+      else
+        resolve_designation(identifier, display)
+      end
+    end
+
+    def resolve_local(term, concept_id)
+      ConceptReference.new(
+        term: term.strip,
+        concept_id: concept_id.strip,
+        source: nil,
+        ref_type: "local",
+      )
+    end
+
+    def resolve_designation(text, display)
+      ConceptReference.new(
+        term: display || text,
+        concept_id: nil,
+        source: nil,
+        ref_type: "designation",
+      )
+    end
+
+    def resolve_iec_urn(urn, display)
+      concept_id = extract_iec_concept_id(urn)
+
+      ConceptReference.new(
+        term: display || "",
+        concept_id: concept_id,
+        source: "urn:iec:std:iec:60050",
+        ref_type: "urn",
+      )
+    end
+
+    def resolve_iso_urn(urn, display)
+      if (m = urn.match(/\Aurn:iso:std:iso:(\d+)(?::(.*))?\z/))
+        term_id = extract_term_id_from_urn_tail(m[2])
+        ConceptReference.new(
+          term: display || "",
+          concept_id: term_id,
+          source: "urn:iso:std:iso:#{m[1]}",
+          ref_type: "urn",
+        )
+      end
+    end
+
+    def resolve_generic_urn(urn, display)
+      ConceptReference.new(
+        term: display || "",
+        concept_id: nil,
+        source: urn,
+        ref_type: "urn",
+      )
+    end
+
+    LANG_CODES = Glossarist::LANG_CODES
+
+    private
+
+    def gather_texts(lc_hash)
+      texts = extract_text_fields(lc_hash["definition"])
+      texts << lc_hash["definition"].to_s if lc_hash["definition"].is_a?(String)
+      texts.concat(extract_text_fields(lc_hash["notes"]))
+      texts.concat(extract_text_fields(lc_hash["examples"]))
+      texts
+    end
+
+    def extract_text_fields(items)
+      Array(items).filter_map do |item|
+        item.is_a?(Hash) ? item["content"]&.to_s : item.to_s
+      end
+    end
+
+    def deduplicate(refs)
+      seen = Set.new
+      refs.select do |ref|
+        key = if ref.concept_id
+                [ref.source,
+                 ref.concept_id]
+              else
+                [ref.source, ref.concept_id,
+                 ref.term]
+              end
+        seen.add?(key)
+      end
+    end
+
+    def extract_term_id_from_urn_tail(tail)
+      return "" unless tail
+
+      if (m = tail.match(/term:([\d.,]+)/))
+        m[1].split(",").first
+      elsif (m = tail.match(/sec:([\d.]+)/))
+        m[1]
+      else
+        tail
+      end
+    end
+
+    def extract_iec_concept_id(urn)
+      if (m = urn.match(/::#con-([\d-]+)/))
+        m[1]
+      else
+        segments = urn.split(":")
+        code_part = segments.find { |s| s.start_with?("60050-") }
+        return "" unless code_part
+
+        code_part.delete_prefix("60050-").sub(/-\d{4}-\d{2}\z/, "")
+      end
+    end
+
+    # Unified concept mention pattern: {{...}}
+    register_pattern(
+      name: :concept_mention,
+      regex: /\{\{([^}]+)\}\}/,
+    ) { |ext, content| ext.resolve_mention(content) }
+
+    register_identifier_resolver("urn:iec:std:iec:60050") do |ext, identifier, display|
+      ext.resolve_iec_urn(identifier, display)
+    end
+
+    register_identifier_resolver("urn:iso:std:iso:") do |ext, identifier, display|
+      ext.resolve_iso_urn(identifier, display)
+    end
+
+    register_identifier_resolver("urn:") do |ext, identifier, display|
+      ext.resolve_generic_urn(identifier, display)
+    end
+  end
+end