glossarist 2.5.0 → 2.5.2

data/lib/glossarist/concept_validator.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ConceptValidator
+    LANG_CODES = Glossarist::LANG_CODES
+    VALID_ENTRY_STATUSES = %w[valid superseded withdrawn draft].freeze
+
+    attr_reader :path, :errors, :warnings
+
+    def initialize(path)
+      @path = path
+      @errors = []
+      @warnings = []
+    end
+
+    def validate_all
+      seen_ids = {}
+      file_idx = 0
+
+      ConceptCollector.each_concept(@path) do |concept|
+        fname = concept_file_name(concept, file_idx)
+        validate_concept(concept, fname, seen_ids)
+        file_idx += 1
+      end
+
+      if file_idx.zero?
+        yaml_files = find_yaml_files
+        if yaml_files.any?
+          @errors << "YAML files found but no parseable concepts"
+        end
+      end
+
+      ValidationResult.new(errors: @errors, warnings: @warnings)
+    end
+
+    private
+
+    def find_yaml_files
+      concepts_dir = if File.directory?(File.join(@path, "concepts"))
+                       File.join(@path, "concepts")
+                     else
+                       @path
+                     end
+      Dir.glob(File.join(concepts_dir, "*.yaml"))
+    end
+
+    def concept_file_name(concept, idx)
+      id = concept.data&.id
+      id ? "concept-#{id}.yaml" : "concept-#{idx}.yaml"
+    end
+
+    def validate_concept(concept, fname, seen_ids)
+      validate_id(concept, fname, seen_ids)
+      validate_localizations(concept, fname)
+      validate_entry_statuses(concept, fname)
+    end
+
+    def validate_id(concept, fname, seen_ids)
+      id = concept.data&.id
+      unless id
+        @errors << "#{fname}: missing concept id"
+        return
+      end
+
+      id_str = id.to_s
+      if seen_ids[id_str]
+        @errors << "#{fname}: duplicate id '#{id_str}' (first seen in #{seen_ids[id_str]})"
+      else
+        seen_ids[id_str] = fname
+      end
+    end
+
+    def validate_localizations(concept, fname)
+      l10ns = concept.localizations&.values || []
+      if l10ns.empty?
+        @errors << "#{fname}: no localizations found"
+        return
+      end
+
+      l10ns.each do |l10n|
+        lang = l10n.language_code || "unknown"
+        terms = l10n.data&.terms
+        unless terms.is_a?(Array) && terms.any?
+          @errors << "#{fname}/#{lang}: must have at least 1 term"
+        end
+      end
+    end
+
+    def validate_entry_statuses(concept, fname)
+      (concept.localizations&.values || []).each do |l10n|
+        lang = l10n.language_code || "unknown"
+        status = l10n.data&.entry_status
+        next unless status
+
+        unless VALID_ENTRY_STATUSES.include?(status)
+          @errors << "#{fname}/#{lang}: invalid entry_status '#{status}' (expected one of: #{VALID_ENTRY_STATUSES.join(', ')})"
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/custom_locality.rb

@@ -8,7 +8,7 @@ module Glossarist
     # @return [String]
     attribute :value, :string
 
-    yaml do
+    key_value do
       map :name, to: :name
       map :value, to: :value
     end

data/lib/glossarist/dataset_validator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class DatasetValidator
+    def validate(path, strict: false, reference_path: nil)
+      result = validate_path(path)
+
+      if reference_path
+        ref_result = validate_cross_references(path, reference_path)
+        result.merge(ref_result)
+      end
+
+      result
+    end
+
+    private
+
+    def validate_path(path)
+      if File.extname(path).downcase == ".gcr"
+        validate_gcr(path)
+      else
+        validate_directory(path)
+      end
+    end
+
+    def validate_gcr(path)
+      GcrValidator.new.validate(path)
+    end
+
+    def validate_directory(path)
+      ConceptValidator.new(path).validate_all
+    end
+
+    def validate_cross_references(path, reference_path)
+      extractor = ReferenceExtractor.new
+      resolver = build_resolver(reference_path)
+
+      if File.extname(path).downcase == ".gcr"
+        validate_gcr_refs(resolver, path, extractor)
+      else
+        validate_directory_refs(resolver, path, extractor)
+      end
+    end
+
+    def build_resolver(reference_path)
+      resolver = ReferenceResolver.new
+      Dir.glob(File.join(reference_path, "*.gcr")).each do |gcr_path|
+        pkg = GcrPackage.load(gcr_path)
+        uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
+        resolver.register_package(pkg, uri_prefix: uri_prefix)
+      end
+      resolver
+    end
+
+    def validate_gcr_refs(resolver, path, extractor)
+      pkg = GcrPackage.load(path)
+      uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
+      resolver.register_self(pkg.concepts)
+      resolver.register_package(pkg, uri_prefix: uri_prefix)
+      resolver.validate_all(pkg, extractor: extractor)
+    end
+
+    def validate_directory_refs(resolver, path, extractor)
+      concepts = ConceptCollector.collect(path)
+      resolver.register_self(concepts)
+      resolver.validate_all(concepts, extractor: extractor)
+    end
+  end
+end

data/lib/glossarist/designation/abbreviation.rb

@@ -8,7 +8,7 @@ module Glossarist
         attribute name.to_sym, :boolean
       end
 
-      yaml do
+      key_value do
         map :international, to: :international
         map :type, to: :type, render_default: true
         Glossarist::GlossaryDefinition::ABBREVIATION_TYPES.each do |name|

data/lib/glossarist/designation/base.rb

@@ -7,7 +7,7 @@ module Glossarist
                 values: Glossarist::GlossaryDefinition::DESIGNATION_BASE_NORMATIVE_STATUSES
       attribute :type, :string
 
-      yaml do
+      key_value do
         map :type, to: :type
         map %i[normative_status normativeStatus], to: :normative_status
         map %i[geographical_area geographicalArea], to: :geographical_area
@@ -18,14 +18,13 @@ module Glossarist
         type = hash["type"]
 
         if type.nil? || /\w/ !~ type
-          raise ArgumentError, "designation type is missing"
+          type = infer_designation_type(hash)
+          hash["type"] = type
         end
 
         if self == Base
-          # called on Base class, delegate it to proper subclass
           SERIALIZED_TYPES[type].of_yaml(hash)
         else
-          # called on subclass, instantiate object
           unless SERIALIZED_TYPES[self] == type
             raise ArgumentError, "unexpected designation type: #{type}"
           end
@@ -33,6 +32,14 @@ module Glossarist
           super
         end
       end
+
+      def self.infer_designation_type(hash)
+        if hash["international"] || hash["abbreviation_type"]
+          "symbol"
+        else
+          "expression"
+        end
+      end
     end
   end
 end

data/lib/glossarist/designation/expression.rb

@@ -13,7 +13,7 @@ module Glossarist
       attribute :grammar_info, GrammarInfo, collection: true
       attribute :type, :string, default: -> { "expression" }
 
-      yaml do
+      key_value do
         map :type, to: :type, render_default: true
         map :prefix, to: :prefix
         map %i[usage_info usageInfo], to: :usage_info

data/lib/glossarist/designation/grammar_info.rb

@@ -8,7 +8,7 @@ module Glossarist
       attribute :part_of_speech, :string,
                 values: Glossarist::GlossaryDefinition::GRAMMAR_INFO_BOOLEAN_ATTRIBUTES
 
-      yaml do
+      key_value do
         map :gender, to: :gender
         map :number, to: :number
 

data/lib/glossarist/designation/graphical_symbol.rb

@@ -6,7 +6,7 @@ module Glossarist
       attribute :text, :string
       attribute :image, :string
 
-      yaml do
+      key_value do
         map :text, to: :text
         map :image, to: :image
       end

data/lib/glossarist/designation/letter_symbol.rb

@@ -5,7 +5,7 @@ module Glossarist
       attribute :language, :string
       attribute :script, :string
 
-      yaml do
+      key_value do
         map :text, to: :text
         map :language, to: :language
         map :script, to: :script

data/lib/glossarist/designation/symbol.rb

@@ -2,9 +2,9 @@ module Glossarist
   module Designation
     class Symbol < Base
       attribute :international, :boolean
-      attribute :type, :string
+      attribute :type, :string, default: -> { "symbol" }
 
-      yaml do
+      key_value do
         map :international, to: :international
         map :type, to: :type, render_default: true
       end

data/lib/glossarist/detailed_definition.rb

@@ -5,7 +5,7 @@ module Glossarist
     attribute :content, :string
     attribute :sources, ConceptSource, collection: true
 
-    yaml do
+    key_value do
       map :content, to: :content
       map :sources, to: :sources
     end

data/lib/glossarist/gcr_metadata.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class GcrMetadata < Lutaml::Model::Serializable
+    attribute :shortname, :string
+    attribute :version, :string
+    attribute :title, :string
+    attribute :description, :string
+    attribute :owner, :string
+    attribute :tags, :string, collection: true
+    attribute :concept_count, :integer
+    attribute :languages, :string, collection: true
+    attribute :created_at, :string
+    attribute :glossarist_version, :string
+    attribute :schema_version, :string, default: -> { "1.0.0" }
+    attribute :statistics, GcrStatistics
+    attribute :homepage, :string
+    attribute :repository, :string
+    attribute :license, :string
+    attribute :uri_prefix, :string
+    attribute :concept_uri_template, :string
+    attribute :external_references, :hash, collection: true
+
+    key_value do
+      map :shortname, to: :shortname
+      map :version, to: :version
+      map :title, to: :title
+      map :description, to: :description
+      map :owner, to: :owner
+      map :tags, to: :tags
+      map :concept_count, to: :concept_count
+      map :languages, to: :languages
+      map :created_at, to: :created_at
+      map :glossarist_version, to: :glossarist_version
+      map :schema_version, to: :schema_version
+      map :statistics, to: :statistics
+      map :homepage, to: :homepage
+      map :repository, to: :repository
+      map :license, to: :license
+      map :uri_prefix, to: :uri_prefix
+      map :concept_uri_template, to: :concept_uri_template
+      map :external_references, to: :external_references
+    end
+
+    def self.from_concepts(concepts, register_data: nil, options: {})
+      stats = GcrStatistics.from_concepts(concepts)
+      new(
+        shortname: options[:shortname] || register_data&.dig("shortname") || register_data&.dig("id"),
+        version: options[:version] || register_data&.dig("version"),
+        title: options[:title] || register_data&.dig("name"),
+        description: options[:description] || register_data&.dig("description"),
+        owner: options[:owner],
+        tags: options[:tags] || [],
+        concept_count: concepts.length,
+        languages: stats.languages,
+        created_at: Time.now.utc.iso8601,
+        glossarist_version: Glossarist::VERSION,
+        schema_version: register_data&.dig("schema_version") || SchemaMigration::CURRENT_SCHEMA_VERSION,
+        statistics: stats,
+        uri_prefix: options[:uri_prefix],
+        concept_uri_template: options[:concept_uri_template],
+        external_references: derive_external_references(concepts),
+      )
+    end
+
+    def self.derive_external_references(concepts)
+      sources = Set.new
+      concepts.each do |concept|
+        concept.localizations.each do |l10n|
+          l10n.data.references&.each do |ref|
+            src = ref.source
+            sources.add(src) if src && !src.empty?
+          end
+        end
+      end
+      sources.map { |uri| { "uri" => uri } }
+    end
+
+    def [](key)
+      to_yaml_hash[key]
+    end
+
+    def dig(*keys)
+      to_yaml_hash.dig(*keys)
+    end
+  end
+end

data/lib/glossarist/gcr_package.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require "zip"
+require "fileutils"
+
+module Glossarist
+  class GcrPackage
+    attr_reader :zip_path, :metadata, :concepts
+
+    def initialize(zip_path)
+      @zip_path = zip_path
+      @metadata = nil
+      @concepts = []
+    end
+
+    def self.create(concepts:, metadata:, output_path:, register_data: nil)
+      FileUtils.mkdir_p(File.dirname(output_path))
+      package = new(output_path)
+      package.send(:write, concepts, metadata, register_data)
+      package
+    end
+
+    def self.load(zip_path)
+      package = new(zip_path)
+      package.send(:read)
+      package
+    end
+
+    def self.create_from_directory(dir, output:, shortname:, version:, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+                                  title: nil, description: nil, owner: nil,
+                                  tags: [], register_yaml: nil,
+                                  uri_prefix: nil, concept_uri_template: nil,
+                                  streaming: false)
+      dir = File.expand_path(dir)
+
+      if streaming
+        create_streaming(dir, output: output, shortname: shortname, version: version,
+                              title: title, description: description, owner: owner,
+                              tags: tags, register_yaml: register_yaml,
+                              uri_prefix: uri_prefix,
+                              concept_uri_template: concept_uri_template)
+      else
+        create_batch(dir, output: output, shortname: shortname, version: version,
+                          title: title, description: description, owner: owner,
+                          tags: tags, register_yaml: register_yaml,
+                          uri_prefix: uri_prefix,
+                          concept_uri_template: concept_uri_template)
+      end
+    end
+
+    def validate
+      GcrValidator.new.validate(@zip_path)
+    end
+
+    private
+
+    def write(concepts, metadata, register_data)
+      Zip::File.open(@zip_path, create: true) do |zf|
+        zf.get_output_stream("metadata.yaml") do |f|
+          f.write(metadata.to_yaml)
+        end
+
+        if register_data
+          zf.get_output_stream("register.yaml") do |f|
+            f.write(register_data.to_yaml)
+          end
+        end
+
+        concepts.each do |mc|
+          write_concept(zf, mc)
+        end
+      end
+    end
+
+    def write_concept(zip_file, concept)
+      termid = concept.data.id.to_s
+      doc = ConceptDocument.from_managed_concept(concept)
+      zip_file.get_output_stream("concepts/#{termid}.yaml") do |f|
+        f.write(doc.to_yamls)
+      end
+    end
+
+    def read
+      @concepts = []
+
+      Zip::File.open(@zip_path) do |zf|
+        if (entry = zf.find_entry("metadata.yaml"))
+          @metadata = GcrMetadata.from_yaml(entry.get_input_stream.read)
+        end
+
+        zf.entries.each do |entry|
+          next unless entry.name.start_with?("concepts/") && entry.name.end_with?(".yaml")
+
+          raw = entry.get_input_stream.read
+          doc = ConceptDocument.from_yamls(raw)
+          @concepts << doc.to_managed_concept
+        end
+      end
+    end
+
+    class << self
+      private
+
+      def create_batch(dir, output:, shortname:, version:, **opts)
+        concepts = ConceptCollector.collect(dir)
+        if concepts.empty?
+          raise ArgumentError,
+                "No concept files found in #{dir}"
+        end
+
+        enricher = ConceptEnricher.new
+        enricher.inject_references(concepts)
+        if opts[:concept_uri_template]
+          enricher.apply_uri_template(concepts,
+                                      opts[:concept_uri_template])
+        end
+
+        register_data = load_register_data(opts[:register_yaml], dir)
+        metadata = build_metadata(concepts, shortname: shortname, version: version,
+                                            register_data: register_data, **opts)
+
+        create(
+          concepts: concepts,
+          metadata: metadata,
+          register_data: register_data,
+          output_path: File.expand_path(output),
+        )
+      end
+
+      def create_streaming(dir, output:, shortname:, version:, **opts) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
+        enricher = ConceptEnricher.new
+        output_path = File.expand_path(output)
+        FileUtils.mkdir_p(File.dirname(output_path))
+
+        register_data = load_register_data(opts[:register_yaml], dir)
+        concept_count = 0
+        languages = Set.new
+
+        Zip::OutputStream.open(output_path) do |zos|
+          if register_data
+            zos.put_next_entry("register.yaml")
+            zos.write(register_data.to_yaml)
+          end
+
+          ConceptCollector.each_concept(dir) do |mc|
+            enricher.inject_references([mc])
+            if opts[:concept_uri_template]
+              enricher.apply_uri_template([mc],
+                                          opts[:concept_uri_template])
+            end
+
+            mc.localizations.each do |l10n|
+              languages << l10n.language_code if l10n.language_code
+            end
+            concept_count += 1
+
+            termid = mc.data.id.to_s
+            doc = ConceptDocument.from_managed_concept(mc)
+            zos.put_next_entry("concepts/#{termid}.yaml")
+            zos.write(doc.to_yamls)
+          end
+
+          if concept_count.zero?
+            raise ArgumentError,
+                  "No concept files found in #{dir}"
+          end
+
+          metadata = build_streaming_metadata(concept_count, languages,
+                                              shortname: shortname, version: version,
+                                              register_data: register_data, **opts)
+          zos.put_next_entry("metadata.yaml")
+          zos.write(metadata.to_yaml)
+        end
+
+        new(output_path)
+      end
+
+      def build_streaming_metadata(concept_count, languages, shortname:, version:, # rubocop:disable Metrics/ParameterLists
+                                   register_data: nil, **opts)
+        GcrMetadata.new(
+          shortname: shortname,
+          version: version,
+          title: opts[:title],
+          description: opts[:description],
+          owner: opts[:owner],
+          tags: opts[:tags] || [],
+          concept_count: concept_count,
+          languages: languages.sort,
+          created_at: Time.now.utc.iso8601,
+          glossarist_version: Glossarist::VERSION,
+          schema_version: register_data&.dig("schema_version") || SchemaMigration::CURRENT_SCHEMA_VERSION,
+          uri_prefix: opts[:uri_prefix],
+          concept_uri_template: opts[:concept_uri_template],
+        )
+      end
+
+      def build_metadata(concepts, shortname:, version:, register_data: nil,
+**opts)
+        GcrMetadata.from_concepts(
+          concepts,
+          register_data: register_data,
+          options: {
+            shortname: shortname,
+            version: version,
+            title: opts[:title],
+            description: opts[:description],
+            owner: opts[:owner],
+            tags: opts[:tags],
+            uri_prefix: opts[:uri_prefix],
+            concept_uri_template: opts[:concept_uri_template],
+          },
+        )
+      end
+
+      def load_register_data(register_yaml_path, dir)
+        path = register_yaml_path || File.join(dir, "register.yaml")
+        return nil unless File.exist?(path)
+
+        RegisterData.from_file(path)
+      end
+    end
+  end
+end

data/lib/glossarist/gcr_statistics.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class GcrStatistics < Lutaml::Model::Serializable
+    attribute :total_concepts, :integer
+    attribute :languages, :string, collection: true
+    attribute :concepts_by_status, :hash
+    attribute :concepts_with_definitions, :integer
+    attribute :concepts_with_sources, :integer
+
+    key_value do
+      map :total_concepts, to: :total_concepts
+      map :languages, to: :languages
+      map :concepts_by_status, to: :concepts_by_status
+      map :concepts_with_definitions, to: :concepts_with_definitions
+      map :concepts_with_sources, to: :concepts_with_sources
+    end
+
+    def self.from_concepts(concepts)
+      l10ns = concepts.flat_map { |c| c.localizations.to_a }
+
+      new(
+        total_concepts: concepts.length,
+        languages: l10ns.map(&:language_code).compact.sort.uniq,
+        concepts_by_status: l10ns.map(&:entry_status).compact.tally,
+        concepts_with_definitions: count_with(l10ns, :definition),
+        concepts_with_sources: count_with(l10ns, :sources),
+      )
+    end
+
+    def self.count_with(l10ns, attr)
+      l10ns.count { |l| l.data.send(attr)&.any? }
+    end
+  end
+end