glossarist 2.6.4 → 2.6.5

data/lib/glossarist/dataset_validator.rb

@@ -3,43 +3,53 @@
 module Glossarist
   class DatasetValidator
     def validate(path, strict: false, reference_path: nil)
-      result = validate_path(path)
+      if File.extname(path).downcase == ".gcr"
+        validate_gcr(path, reference_path: reference_path)
+      else
+        validate_directory(path, reference_path: reference_path)
+      end
+    end
+
+    private
+
+    def validate_gcr(path, reference_path: nil)
+      result = GcrValidator.new.validate(path)
 
       if reference_path
-        ref_result = validate_cross_references(path, reference_path)
+        ref_result = validate_gcr_cross_references(path, reference_path)
         result.merge(ref_result)
       end
 
       result
     end
 
-    private
+    def validate_directory(path, reference_path: nil)
+      result = ConceptValidator.new(path).validate_all
 
-    def validate_path(path)
-      if File.extname(path).downcase == ".gcr"
-        validate_gcr(path)
-      else
-        validate_directory(path)
+      if reference_path
+        ref_result = validate_directory_cross_references(path, reference_path)
+        result.merge(ref_result)
       end
-    end
 
-    def validate_gcr(path)
-      GcrValidator.new.validate(path)
+      result
     end
 
-    def validate_directory(path)
-      ConceptValidator.new(path).validate_all
+    def validate_gcr_cross_references(path, reference_path)
+      extractor = ReferenceExtractor.new
+      resolver = build_resolver(reference_path)
+      pkg = GcrPackage.load(path)
+      uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
+      resolver.register_self(pkg.concepts)
+      resolver.register_package(pkg, uri_prefix: uri_prefix)
+      resolver.validate_all(pkg, extractor: extractor)
     end
 
-    def validate_cross_references(path, reference_path)
+    def validate_directory_cross_references(path, reference_path)
       extractor = ReferenceExtractor.new
       resolver = build_resolver(reference_path)
-
-      if File.extname(path).downcase == ".gcr"
-        validate_gcr_refs(resolver, path, extractor)
-      else
-        validate_directory_refs(resolver, path, extractor)
-      end
+      concepts = ConceptCollector.collect(path)
+      resolver.register_self(concepts)
+      resolver.validate_all(concepts, extractor: extractor)
     end
 
     def build_resolver(reference_path)
@@ -51,19 +61,5 @@ module Glossarist
       end
       resolver
     end
-
-    def validate_gcr_refs(resolver, path, extractor)
-      pkg = GcrPackage.load(path)
-      uri_prefix = pkg.metadata&.dig("uri_prefix") || pkg.metadata&.dig("shortname")
-      resolver.register_self(pkg.concepts)
-      resolver.register_package(pkg, uri_prefix: uri_prefix)
-      resolver.validate_all(pkg, extractor: extractor)
-    end
-
-    def validate_directory_refs(resolver, path, extractor)
-      concepts = ConceptCollector.collect(path)
-      resolver.register_self(concepts)
-      resolver.validate_all(concepts, extractor: extractor)
-    end
   end
 end

data/lib/glossarist/gcr_validator.rb

@@ -4,7 +4,7 @@ require "zip"
 
 module Glossarist
   class GcrValidator
-    def validate(zip_path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def validate(zip_path)
       result = ValidationResult.new
 
       unless File.exist?(zip_path)
@@ -13,123 +13,48 @@ module Glossarist
       end
 
       begin
-        Zip::File.open(zip_path) do |zip_file|
-          validate_zip_contents(zip_file, result)
-        end
+        zip_entries = Zip::File.open(zip_path) { |zf| zf.entries.to_set(&:name) }
       rescue StandardError => e
         result.add_error("Failed to read ZIP: #{e.message}")
+        return result
       end
 
-      result
-    end
-
-    private
-
-    def validate_zip_contents(zip_file, result) # rubocop:disable Metrics/AbcSize
-      unless zip_file.find_entry("metadata.yaml")
+      unless zip_entries.include?("metadata.yaml")
         result.add_error("Missing metadata.yaml")
-        return
-      end
-
-      metadata = GcrMetadata.from_yaml(
-        zip_file.find_entry("metadata.yaml").get_input_stream.read,
-      )
-      validate_metadata(metadata, result)
-
-      concept_entries = zip_file.entries.select do |e|
-        e.name.start_with?("concepts/") && e.name.end_with?(".yaml")
-      end
-      if concept_entries.empty?
-        result.add_error("No concept files found in concepts/")
-      end
-
-      concept_entries.each do |entry|
-        validate_concept_entry(entry, metadata, result)
-      end
-
-      validate_assets(zip_file, result)
-    end
-
-    def validate_metadata(metadata, result)
-      unless metadata&.concept_count
-        result.add_error("metadata.yaml missing required fields (concept_count)")
-      end
-
-      unless metadata&.shortname
-        result.add_error("metadata.yaml missing shortname")
+        return result
       end
 
-      unless metadata&.version
-        result.add_error("metadata.yaml missing version")
+      begin
+        context = Validation::Rules::GcrContext.new(zip_path)
+      rescue StandardError => e
+        result.add_error("Failed to load GCR: #{e.message}")
+        return result
       end
-    end
 
-    def validate_concept_entry(entry, metadata, result) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      raw = entry.get_input_stream.read
-      doc = ConceptDocument.from_yamls(raw)
-    rescue Psych::SyntaxError => e
-      result.add_error("#{entry.name}: invalid YAML at line #{e.line}: #{e.message}")
-    rescue StandardError => e
-      result.add_error("#{entry.name}: parse error: #{e.message}")
-    else
-      concept = doc.concept
-      unless concept&.data&.id
-        result.add_error("#{entry.name}: document 0 missing data.identifier")
-      end
+      # Collection-level rules (metadata, structure, integrity)
+      collection_rules = Validation::Rules::Registry.for_scope(:collection)
+      collection_rules.each do |rule|
+        next unless rule.applicable?(context)
 
-      localizations = doc.localizations
-      if localizations.empty?
-        result.add_error("#{entry.name}: expected at least 1 localization document")
-      else
-        localizations.each_with_index do |l10n, idx|
-          unless l10n&.language_code
-            result.add_error("#{entry.name}: document #{idx + 1} missing data.language_code")
-          end
-        end
+        rule.check(context).each { |i| result.add_issue(i) }
       end
 
-      validate_concept_uri(entry, concept, metadata, result)
-    end
-
-    def validate_concept_uri(entry, concept, metadata, result) # rubocop:disable Metrics/CyclomaticComplexity
-      concept_uri = concept&.data&.uri
-      template = metadata&.concept_uri_template
-      uri_prefix = metadata&.uri_prefix
+      # Per-concept rules
+      concept_rules = Validation::Rules::Registry.for_scope(:concept)
+      context.concepts.each_with_index do |concept, idx|
+        fname = concept.data&.id ? "concepts/#{concept.data.id}.yaml" : "concepts/concept-#{idx}.yaml"
+        concept_context = Validation::Rules::ConceptContext.new(
+          concept, file_name: fname, collection_context: context
+        )
 
-      if concept_uri.nil? && template.nil? && uri_prefix.nil?
-        result.add_warning("#{entry.name}: no concept URI (data.uri) and no concept_uri_template or uri_prefix in metadata")
-      end
-    end
+        concept_rules.each do |rule|
+          next unless rule.applicable?(concept_context)
 
-    def validate_assets(zip_file, result)
-      GcrPackage::DATASET_ASSETS.each do |asset|
-        case asset[:type]
-        when :file
-          validate_file_asset_entry(zip_file, asset[:path], result)
-        when :directory
-          validate_directory_asset(zip_file, asset[:path], result)
+          rule.check(concept_context).each { |i| result.add_issue(i) }
         end
       end
-    end
-
-    def validate_file_asset_entry(zip_file, path, result)
-      entry = zip_file.find_entry(path)
-      return unless entry
-
-      YAML.safe_load(entry.get_input_stream.read)
-    rescue Psych::SyntaxError => e
-      result.add_error("#{path}: invalid YAML at line #{e.line}: #{e.message}")
-    end
 
-    def validate_directory_asset(zip_file, dir_path, result)
-      dir_entries = zip_file.entries.select do |e|
-        e.name.start_with?("#{dir_path}/")
-      end
-      return unless dir_entries.any? && dir_entries.all? do |e|
-        e.name.end_with?("/")
-      end
-
-      result.add_warning("#{dir_path}/ directory is empty")
+      result
     end
   end
 end

data/lib/glossarist/reference_extractor.rb

@@ -152,6 +152,65 @@ module Glossarist
 
     LANG_CODES = Glossarist::LANG_CODES
 
+    # Extract asset references from model attributes (NonVerbRep, GraphicalSymbol).
+    def extract_asset_refs_from_concept(concept)
+      refs = []
+
+      concept.localizations.each do |l10n|
+        nvr = l10n.non_verb_rep
+        if nvr.is_a?(String) && !nvr.strip.empty?
+          nvr.strip.split.each do |p|
+            refs << AssetReference.new(path: p) unless p.empty?
+          end
+        end
+
+        (l10n.data&.terms || []).each do |term|
+          if term.is_a?(Designation::GraphicalSymbol) && term.image && !term.image.strip.empty?
+            refs << AssetReference.new(path: term.image.strip)
+          end
+        end
+      end
+
+      refs
+    end
+
+    # Extract bibliographic xrefs from model-level source citations.
+    def extract_bib_refs_from_concept(concept)
+      refs = []
+      concept.localizations.each do |l10n|
+        gather_all_sources(l10n).each do |source|
+          origin = source.origin
+          next unless origin
+
+          if origin.text && !origin.text.strip.empty?
+            refs << BibliographicReference.new(anchor: origin.text)
+          end
+
+          next unless origin.source && origin.id
+
+          key = "#{origin.source} #{origin.id}"
+          refs << BibliographicReference.new(anchor: key)
+          refs << BibliographicReference.new(anchor: origin.id.to_s)
+        end
+      end
+      refs
+    end
+
+    # Extract all reference types from a managed concept.
+    def extract_all_from_managed_concept(concept)
+      concept_refs = extract_from_managed_concept(concept)
+      asset_refs = extract_asset_refs_from_concept(concept)
+      concept_refs + asset_refs
+    end
+
+    def resolve_asciidoc_xref(target)
+      BibliographicReference.new(anchor: target.strip)
+    end
+
+    def resolve_image_ref(path)
+      AssetReference.new(path: path.strip)
+    end
+
     private
 
     def gather_texts(lc_hash)
@@ -170,16 +229,7 @@ module Glossarist
 
     def deduplicate(refs)
       seen = Set.new
-      refs.select do |ref|
-        key = if ref.concept_id
-                [ref.source,
-                 ref.concept_id]
-              else
-                [ref.source, ref.concept_id,
-                 ref.term]
-              end
-        seen.add?(key)
-      end
+      refs.select { |ref| seen.add?(ref.dedup_key) }
     end
 
     def extract_term_id_from_urn_tail(tail)
@@ -212,6 +262,18 @@ module Glossarist
       regex: /\{\{([^}]+)\}\}/,
     ) { |ext, content| ext.resolve_mention(content) }
 
+    # AsciiDoc cross-references: <<anchor>> or <<anchor,display text>>
+    register_pattern(
+      name: :asciidoc_xref,
+      regex: /<<([^,>\n]+?)(?:,[^>\n]*)?>>/,
+    ) { |ext, target| ext.resolve_asciidoc_xref(target) }
+
+    # Image references: image::path[] or image:path[]
+    register_pattern(
+      name: :asciidoc_image,
+      regex: /image::?([^\[\]]+)\[/,
+    ) { |ext, path| ext.resolve_image_ref(path) }
+
     register_identifier_resolver("urn:iec:std:iec:60050") do |ext, identifier, display|
       ext.resolve_iec_urn(identifier, display)
     end
@@ -223,5 +285,13 @@ module Glossarist
     register_identifier_resolver("urn:") do |ext, identifier, display|
       ext.resolve_generic_urn(identifier, display)
     end
+
+    def gather_all_sources(l10n)
+      sources = Array(l10n.data&.sources)
+      sources += Array((l10n.data&.definition || []).flat_map(&:sources).compact)
+      sources += Array((l10n.data&.notes || []).flat_map(&:sources).compact)
+      sources += Array((l10n.data&.examples || []).flat_map(&:sources).compact)
+      sources
+    end
   end
 end

data/lib/glossarist/reference_resolver.rb

@@ -153,6 +153,7 @@ module Glossarist
     def extract_refs(concept, extractor)
       if concept.is_a?(ManagedConcept)
         extractor.extract_from_managed_concept(concept)
+          .grep(ConceptReference)
       else
         extractor.extract_from_concept_hash(concept)
       end

data/lib/glossarist/validation/asset_index.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "set"
+require "zip"
+
+module Glossarist
+  module Validation
+    class AssetIndex
+      IMAGE_TERMS = %w[id ref text anchor].freeze
+      private_constant :IMAGE_TERMS
+
+      attr_reader :paths
+
+      def initialize
+        @paths = Set.new
+      end
+
+      def register(path)
+        @paths.add(normalize_path(path))
+      end
+
+      def resolve?(path)
+        @paths.include?(normalize_path(path))
+      end
+
+      def each_path(&block)
+        @paths.each(&block)
+      end
+
+      def self.build_from_directory(dataset_path)
+        index = new
+        index_image_files(index, dataset_path)
+        index_model_assets(index, dataset_path)
+        index
+      end
+
+      def self.build_from_zip(zip_path)
+        index = new
+        index_zip_images(index, zip_path)
+        index_zip_concept_assets(index, zip_path)
+        index
+      end
+
+      private
+
+      def normalize_path(path)
+        path.to_s.delete_prefix("/")
+      end
+
+      class << self
+        private
+
+        def index_image_files(index, dataset_path)
+          images_dir = File.join(dataset_path, "images")
+          return unless File.directory?(images_dir)
+
+          base = File.expand_path(dataset_path)
+          Dir.glob(File.join(images_dir, "**", "*")).each do |file|
+            next unless File.file?(file)
+
+            relative = file.sub("#{base}/", "")
+            index.register(relative)
+          end
+        end
+
+        def index_model_assets(index, dataset_path)
+          concepts = ConceptCollector.collect(dataset_path)
+          index_concept_assets(index, concepts)
+        end
+
+        def index_zip_images(index, zip_path)
+          Zip::File.open(zip_path) do |zf|
+            zf.entries.each do |entry|
+              next if entry.name.end_with?("/")
+              next unless entry.name.start_with?("images/")
+
+              index.register(entry.name)
+            end
+          end
+        end
+
+        def index_zip_concept_assets(index, zip_path)
+          pkg = GcrPackage.load(zip_path)
+          index_concept_assets(index, pkg.concepts)
+        end
+
+        def index_concept_assets(index, concepts)
+          concepts.each do |concept|
+            concept.localizations.each do |l10n|
+              register_non_verb_rep(index, l10n)
+              register_graphical_symbols(index, l10n)
+            end
+          end
+        end
+
+        def register_non_verb_rep(index, l10n)
+          nvr = l10n.non_verb_rep
+          return unless nvr.is_a?(String) && !nvr.strip.empty?
+
+          nvr.strip.split.each { |p| index.register(p) }
+        end
+
+        def register_graphical_symbols(index, l10n)
+          (l10n.data&.terms || []).each do |term|
+            next unless term.is_a?(Designation::GraphicalSymbol) && term.image
+
+            index.register(term.image)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/validation/bibliography_index.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Validation
+    class BibliographyIndex
+      BIB_ENTRY_KEYS = %w[id ref text anchor].freeze
+      private_constant :BIB_ENTRY_KEYS
+
+      attr_reader :entries
+
+      def initialize
+        @entries = {}
+      end
+
+      def register(anchor, source = nil)
+        @entries[normalize_anchor(anchor)] = { anchor: anchor, source: source }
+      end
+
+      def resolve?(anchor)
+        @entries.key?(normalize_anchor(anchor))
+      end
+
+      def anchors
+        @entries.keys
+      end
+
+      def each_entry(&block)
+        @entries.each_value(&block)
+      end
+
+      def self.build_from_concepts(concepts, dataset_path: nil,
+bibliography_yaml: nil)
+        index = new
+
+        concepts.each { |concept| index_concept_sources(index, concept) }
+
+        yaml = bibliography_yaml || read_bibliography_file(dataset_path)
+        index_bibliography_yaml(index, yaml) if yaml
+
+        index
+      end
+
+      private
+
+      def normalize_anchor(anchor)
+        anchor.to_s.gsub(/[ \/:]/, "_").gsub(/__+/, "_")
+      end
+
+      class << self
+        private
+
+        def read_bibliography_file(dataset_path)
+          return nil unless dataset_path
+
+          bib_path = File.join(dataset_path, "bibliography.yaml")
+          File.exist?(bib_path) ? File.read(bib_path) : nil
+        end
+
+        def index_concept_sources(index, concept)
+          concept.localizations.each do |l10n|
+            index_l10n_sources(index, l10n)
+          end
+        end
+
+        def index_l10n_sources(index, l10n)
+          data = l10n.data
+          return unless data
+
+          register_source_collection(index, data.sources)
+          register_source_collection(index,
+                                     data.definition&.flat_map(&:sources))
+          register_source_collection(index, data.notes&.flat_map(&:sources))
+          register_source_collection(index, data.examples&.flat_map(&:sources))
+        end
+
+        def register_source_collection(index, sources)
+          Array(sources).compact.each { |s| register_source(index, s) }
+        end
+
+        def register_source(index, source)
+          origin = source.origin
+          return unless origin
+
+          register_origin_text(index, origin)
+          register_origin_ref(index, origin)
+        end
+
+        def register_origin_text(index, origin)
+          return unless origin.text && !origin.text.strip.empty?
+
+          index.register(origin.text, origin)
+        end
+
+        def register_origin_ref(index, origin)
+          return unless origin.source && origin.id
+
+          key = "#{origin.source} #{origin.id}"
+          index.register(key, origin)
+          index.register(origin.id.to_s, origin)
+        end
+
+        def index_bibliography_yaml(index, yaml_content)
+          data = YAML.safe_load(yaml_content)
+          return unless data.is_a?(Hash) || data.is_a?(Array)
+
+          entries = data.is_a?(Hash) ? data.values : data
+          entries.each do |entry|
+            next unless entry.is_a?(Hash)
+
+            BIB_ENTRY_KEYS.each do |key|
+              val = entry[key]
+              index.register(val.to_s, entry) if val && !val.to_s.strip.empty?
+            end
+          end
+        rescue Psych::SyntaxError, Psych::DisallowedClass
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/validation/rules/asciidoc_xref_rule.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Validation
+    module Rules
+      class AsciidocXrefRule < Base
+        def code = "GLS-102"
+        def category = :references
+        def severity = "warning"
+        def scope = :concept
+
+        def applicable?(context)
+          context.concept.localizations&.any?
+        end
+
+        def check(context)
+          concept = context.concept
+          fname = context.file_name
+          extractor = ReferenceExtractor.new
+          issues = []
+
+          concept.localizations.each do |l10n|
+            lang = l10n.language_code || "unknown"
+            texts = extract_texts(l10n)
+
+            texts.each do |text|
+              next unless text
+              refs = extractor.extract_from_text(text)
+              refs.each do |ref|
+                next unless ref.is_a?(BibliographicReference)
+                next if context.bibliography_index.resolve?(ref.anchor)
+
+                issues << issue(
+                  "unresolved bibliography reference <<#{ref.anchor}>>",
+                  code: code, severity: severity,
+                  location: "#{fname}/#{lang}",
+                  suggestion: "add '#{ref.anchor}' as a source, " \
+                              "or verify it exists in bibliography.yaml",
+                )
+              end
+            end
+          end
+
+          issues
+        end
+
+        private
+
+        def extract_texts(l10n)
+          texts = []
+          (l10n.data&.definition || []).each { |d| texts << d.content if d.content }
+          (l10n.data&.notes || []).each { |n| texts << n.content if n.content }
+          (l10n.data&.examples || []).each { |e| texts << e.content if e.content }
+          texts
+        end
+      end
+    end
+  end
+end
+