glossarist 2.8.7 → 2.8.10

data/lib/glossarist/glossary_store.rb

@@ -2,16 +2,33 @@
 
 module Glossarist
   class GlossaryStore
-    attr_reader :package
+    attr_reader :package, :localized_concepts_dir_name
 
     def initialize
       @package = nil
       @concept_document_class = V3::ConceptDocument
+      @v1_concepts = nil
+      @localized_concepts_dir_name = nil
     end
 
     # ── Load ──
 
     def load_directory(path, format: nil)
+      if v1_dataset?(path)
+        load_v1_fallback(path)
+        return self
+      end
+
+      if legacy_managed_layout?(path)
+        load_legacy_managed(path)
+        return self
+      end
+
+      if grouped_at_root?(path)
+        load_grouped_at_root(path)
+        return self
+      end
+
       metadata = load_metadata_from_directory(path)
       @concept_document_class = resolve_concept_document_class(metadata)
 
@@ -64,9 +81,23 @@ module Glossarist
     # ── Concepts ──
 
     def concepts
+      return @v1_concepts if @v1_concepts
+
       @package.models_for(@concept_document_class).map(&:to_managed_concept)
     end
 
+    def each_concept(&block)
+      return enum_for(:each_concept) unless block
+
+      if @v1_concepts
+        @v1_concepts.each(&block)
+      else
+        @package.models_for(@concept_document_class).each do |doc|
+          yield doc.to_managed_concept
+        end
+      end
+    end
+
     def concept(uuid)
       doc = @package.fetch_model(@concept_document_class, uuid)
       doc&.to_managed_concept
@@ -194,5 +225,148 @@ module Glossarist
     def apply_metadata(metadata)
       @package.metadata = metadata if metadata && @package
     end
+
+    def load_v1_fallback(path)
+      concepts_dir = File.join(path, "concepts")
+      files = Dir.glob(File.join(concepts_dir, "*.yaml"))
+      @v1_concepts = files.filter_map do |file|
+        v1 = V1::Concept.from_file(file)
+        v1&.to_managed_concept
+      end
+    end
+
+    def legacy_managed_layout?(path)
+      concept_dir = File.join(path, "concept")
+      return false unless File.directory?(concept_dir)
+      return false if File.directory?(File.join(path, "concepts"))
+
+      Dir.glob(File.join(concept_dir, "*.yaml")).any?
+    end
+
+    def load_legacy_managed(path)
+      concept_dir = File.join(path, "concept")
+      lc_dir = find_localized_concepts_dir(path)
+      lc_index = build_lc_index(lc_dir) if lc_dir
+
+      @v1_concepts = []
+      Dir.glob(File.join(concept_dir, "*.yaml")).each do |f|
+        raw = File.read(f, encoding: "utf-8")
+        version = detect_version(raw)
+        doc_class = ConceptDocument.for_version(version)
+        doc = doc_class.from_yamls(raw)
+        mc = doc.concept
+        next unless mc&.data&.id
+
+        load_legacy_localizations(mc, lc_index, version) if lc_index
+        @v1_concepts << mc
+      rescue Psych::SyntaxError => e
+        raise Errors::ParseError.new(filename: f, line: e.line)
+      rescue Lutaml::Model::InvalidFormatError => e
+        raise Errors::ParseError.new(filename: f, message: e.message)
+      rescue Encoding::InvalidByteSequenceError => e
+        raise Errors::LoadError.new(path: f, reason: e.message)
+      end
+    end
+
+    def load_legacy_localizations(managed_concept, lc_index, version = "3")
+      l10n_class = version.to_s == "2" ? V2::LocalizedConcept : LocalizedConcept
+      lc_map = managed_concept.data.localized_concepts || {}
+      lc_map.each_value do |uuid|
+        lc_file = lc_index[uuid]
+        unless lc_file
+          raise Errors::LoadError.new(path: lc_file,
+                                      reason: "Referenced localization #{uuid} not found")
+        end
+
+        l10n = l10n_class.from_yaml(File.read(lc_file, encoding: "utf-8"))
+        l10n.uuid = uuid
+        managed_concept.add_localization(l10n)
+      rescue Errors::LoadError
+        raise
+      rescue Psych::SyntaxError => e
+        raise Errors::ParseError.new(filename: lc_file, line: e.line)
+      rescue Errno::ENOENT
+        raise Errors::LoadError.new(path: lc_file, reason: "File not found")
+      rescue Errno::EACCES
+        raise Errors::LoadError.new(path: lc_file, reason: "Permission denied")
+      end
+    end
+
+    def find_localized_concepts_dir(path)
+      %w[localized_concept localized-concept].each do |name|
+        d = File.join(path, name)
+        if File.directory?(d)
+          @localized_concepts_dir_name = name
+          return d
+        end
+      end
+      nil
+    end
+
+    def build_lc_index(lc_dir)
+      Dir.glob(File.join(lc_dir, "*.{yaml,yml}"))
+        .to_h { |f| [File.basename(f, ".*"), f] }
+    end
+
+    def grouped_at_root?(path)
+      return false if File.directory?(File.join(path, "concepts"))
+      return false if File.directory?(File.join(path, "concept"))
+
+      Dir.glob(File.join(path, "*.yaml")).any? do |f|
+        raw = File.read(f, encoding: "utf-8")
+        hash = YAML.safe_load(raw, permitted_classes: [Date, Time])
+        hash.is_a?(Hash) && hash.key?("data") && hash["data"].is_a?(Hash) &&
+          hash["data"].key?("identifier")
+      rescue Psych::SyntaxError, Encoding::InvalidByteSequenceError
+        false
+      end
+    end
+
+    def load_grouped_at_root(path)
+      @v1_concepts = []
+      Dir.glob(File.join(path, "*.yaml")).each do |f|
+        raw = File.read(f, encoding: "utf-8")
+        version = detect_version(raw)
+        doc_class = ConceptDocument.for_version(version)
+        doc = doc_class.from_yamls(raw)
+        mc = doc.concept
+        next unless mc&.data&.id
+
+        Array(doc.localizations).each { |l10n| mc.add_localization(l10n) }
+        @v1_concepts << mc
+      rescue Psych::SyntaxError => e
+        raise Errors::ParseError.new(filename: f, line: e.line)
+      rescue Lutaml::Model::InvalidFormatError => e
+        raise Errors::ParseError.new(filename: f, message: e.message)
+      rescue Encoding::InvalidByteSequenceError => e
+        raise Errors::LoadError.new(path: f, reason: e.message)
+      end
+    end
+
+    def detect_version(raw)
+      if (m = raw.match(/^schema_version:\s*v?(\d)/))
+        m[1]
+      else
+        "2"
+      end
+    end
+
+    def v1_dataset?(path)
+      concepts_dir = File.join(path, "concepts")
+      return false unless File.directory?(concepts_dir)
+
+      metadata_file = File.join(path, "metadata.yaml")
+      concept_subdir = File.join(concepts_dir, "concept")
+      return false if File.exist?(metadata_file) || File.directory?(concept_subdir)
+
+      sample = Dir.glob(File.join(concepts_dir, "*.yaml")).first
+      return false unless sample
+
+      raw = File.read(sample, encoding: "utf-8")
+      hash = YAML.safe_load(raw, permitted_classes: [Date, Time])
+      hash.is_a?(Hash) && hash.key?("termid")
+    rescue Psych::SyntaxError, Encoding::InvalidByteSequenceError
+      false
+    end
   end
 end

data/lib/glossarist/managed_concept.rb

@@ -19,11 +19,11 @@ module Glossarist
 
     attribute :uuid, :string
 
+    attribute :version, :string
     attribute :schema_version, :string
 
     key_value do
       map :data, to: :data
-      map :id, with: { to: :identifier_to_yaml, from: :identifier_from_yaml }
       map :identifier,
           with: { to: :identifier_to_yaml, from: :identifier_from_yaml }
       map :related, to: :related
@@ -33,7 +33,8 @@ module Glossarist
           with: { from: :date_accepted_from_yaml, to: :date_accepted_to_yaml }
       map :status, to: :status
 
-      map :uuid, to: :uuid, with: { from: :uuid_from_yaml, to: :uuid_to_yaml }
+      map %i[id uuid], to: :uuid,
+                       with: { from: :uuid_from_yaml, to: :uuid_to_yaml }
       map :schema_version, to: :schema_version
     end
 
@@ -143,6 +144,19 @@ module Glossarist
       localization("eng") || localizations.values.first
     end
 
+    def all_sources
+      list = Array(sources)
+      list.concat(Array(data&.sources))
+      localizations.each_value { |l10n| list.concat(l10n.all_sources) }
+      list
+    end
+
+    def find_source_by_id(id)
+      return nil if id.nil? || id.to_s.strip.empty?
+
+      all_sources.find { |source| source.id == id }
+    end
+
     def schema_version
       @schema_version
     end

data/lib/glossarist/managed_concept_collection.rb

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 module Glossarist
   class ManagedConceptCollection
     include Enumerable
 
     attr_accessor :managed_concepts
 
-    def initialize
+    def initialize(file_key: nil)
       @managed_concepts = []
       @managed_concepts_ids = {}
-      @concept_manager = ConceptManager.new
+      @file_key = file_key
     end
 
     def to_h
@@ -33,6 +35,16 @@ module Glossarist
     end
     alias :[] :fetch
 
+    def by_id_and(id, version = nil)
+      return fetch(id) if version.nil?
+
+      @managed_concepts.find do |c|
+        next false unless c.uuid == id || c.uuid == @managed_concepts_ids[id]
+
+        c.version == version
+      end
+    end
+
     # If ManagedConcept with given ID is present in this collection, then
     # returns it. Otherwise, instantiates a new ManagedConcept, adds it to
     # the collection, and returns it.
@@ -62,18 +74,50 @@ module Glossarist
     alias :<< :store
 
     def load_from_files(path)
-      @concept_manager.path = path
-      @concept_manager.load_from_files(collection: self)
+      store = GlossaryStore.new
+      store.load(path)
+      store.concepts.each { |mc| store(mc) }
+      @localized_concepts_path = store.localized_concepts_dir_name || "localized_concept"
     end
 
     def save_to_files(path)
-      @concept_manager.path = path
-      @concept_manager.save_to_files(@managed_concepts)
+      concept_dir = File.join(path, "concept")
+      lc_dir = File.join(path, @localized_concepts_path || "localized_concept")
+      FileUtils.mkdir_p(concept_dir)
+      FileUtils.mkdir_p(lc_dir)
+
+      @managed_concepts.each do |mc|
+        File.write(File.join(concept_dir, "#{file_key(mc)}.yaml"), mc.to_yaml,
+                   encoding: "utf-8")
+
+        mc.localized_concepts.each do |lang, uuid|
+          l10n = mc.localization(lang)
+          next unless l10n
+
+          File.write(File.join(lc_dir, "#{uuid}.yaml"), l10n.to_yaml,
+                     encoding: "utf-8")
+        end
+      end
     end
 
     def save_grouped_concepts_to_files(path)
-      @concept_manager.path = path
-      @concept_manager.save_grouped_concepts_to_files(@managed_concepts)
+      FileUtils.mkdir_p(path)
+
+      @managed_concepts.each do |mc|
+        parts = [mc.to_yaml]
+        mc.localized_concepts.each_key do |lang|
+          l10n = mc.localization(lang)
+          parts << l10n.to_yaml if l10n
+        end
+        File.write(File.join(path, "#{file_key(mc)}.yaml"), parts.join("\n"),
+                   encoding: "utf-8")
+      end
+    end
+
+    private
+
+    def file_key(concept)
+      @file_key ? @file_key.call(concept) : concept.uuid
     end
   end
 end

data/lib/glossarist/reference_extractor.rb

@@ -72,8 +72,8 @@ module Glossarist
 
       if content.include?(",")
         parts = content.split(",", 2)
-        display = parts[0].strip
-        identifier = parts[1].strip
+        identifier = parts[0].strip
+        display = parts[1].strip
         resolve_by_identifier(identifier, display)
       else
         resolve_by_identifier(content, nil)
@@ -114,6 +114,22 @@ module Glossarist
       )
     end
 
+    def resolve_cite_key(identifier, display)
+      cleaned = identifier.delete_prefix("cite:").strip
+      return nil if cleaned.empty?
+
+      if cleaned.start_with?('"') && cleaned.end_with?('"') && cleaned.length >= 2
+        cleaned = cleaned[1..-2].gsub('""', '"')
+      end
+
+      ConceptReference.new(
+        concept_id: cleaned,
+        source: nil,
+        term: display || cleaned,
+        ref_type: "cite",
+      )
+    end
+
     def resolve_iec_urn(urn, display)
       concept_id = extract_iec_concept_id(urn)
 
@@ -273,6 +289,10 @@ module Glossarist
       regex: /image::?([^\[\]]+)\[/,
     ) { |ext, path| ext.resolve_image_ref(path) }
 
+    register_identifier_resolver("cite:") do |ext, identifier, display|
+      ext.resolve_cite_key(identifier, display)
+    end
+
     register_identifier_resolver("urn:iec:std:iec:60050") do |ext, identifier, display|
       ext.resolve_iec_urn(identifier, display)
     end

data/lib/glossarist/reference_resolver.rb

@@ -5,6 +5,7 @@ module Glossarist
     def initialize
       @local_adapter = nil
       @package_adapters = []
+      @bibliography_adapters = []
       @route_adapter = ResolutionAdapter::Route.new
       @remote_adapters = []
     end
@@ -31,13 +32,28 @@ module Glossarist
                                                         endpoint: endpoint)
     end
 
-    def resolve(reference)
+    def register_bibliography(source_id, concepts)
+      @bibliography_adapters << ResolutionAdapter::Bibliography.new(source_id,
+                                                                    concepts)
+    end
+
+    def resolve(reference, concept: nil)
+      if concept && reference.is_a?(ConceptReference) && reference.cite?
+        source = concept.find_source_by_id(reference.concept_id)
+        return source&.origin
+      end
+
       if reference.local?
         return @local_adapter&.resolve(reference)
       end
 
       routed_ref = apply_routes(reference)
 
+      @bibliography_adapters.each do |adapter|
+        result = adapter.resolve(routed_ref)
+        return result if result
+      end
+
       @package_adapters.each do |adapter|
         result = adapter.resolve(routed_ref)
         return result if result
@@ -54,7 +70,8 @@ module Glossarist
     def resolve_all(concept, extractor: nil)
       extractor ||= ReferenceExtractor.new
       refs = extract_refs(concept, extractor)
-      refs.map { |ref| [ref, resolve(ref)] }
+      source_concept = concept.is_a?(ManagedConcept) ? concept : nil
+      refs.map { |ref| [ref, resolve(ref, concept: source_concept)] }
     end
 
     def validate_all(package_or_concepts, extractor: nil, mode: :multi)
@@ -65,9 +82,10 @@ module Glossarist
       concepts.each do |concept|
         refs = extract_refs(concept, extractor)
         termid = extract_termid(concept)
+        source_concept = concept.is_a?(ManagedConcept) ? concept : nil
 
         refs.each do |ref|
-          resolved = resolve(ref)
+          resolved = resolve(ref, concept: source_concept)
           if resolved.nil?
             scope = ref.local? ? "intra-set" : "inter-set (#{ref.source})"
             result.add_warning("#{termid}: Unresolvable #{scope} reference: #{ref.term} -> #{ref.concept_id}")
@@ -98,8 +116,25 @@ module Glossarist
       @package_adapters.map(&:uri_prefix)
     end
 
+    def classify(reference, concept: nil)
+      return "unknown" unless reference.is_a?(ConceptReference)
+
+      resolved = resolve(reference, concept: concept)
+      classify_from_resolution(reference, resolved)
+    end
+
     private
 
+    def classify_from_resolution(reference, resolved)
+      if reference.cite?
+        resolved ? "self-contained-citation" : "unresolved-citation"
+      elsif reference.external?
+        resolved ? "internal-citation" : "external-citation"
+      else
+        resolved ? "same-dataset" : "unresolved"
+      end
+    end
+
     def apply_routes(reference)
       routed = @route_adapter.resolve(reference)
       routed || reference

data/lib/glossarist/resolution_adapter/bibliography.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class ResolutionAdapter
+    class Bibliography < ResolutionAdapter
+      attr_reader :source_id, :concepts
+
+      def initialize(source_id, concepts)
+        super()
+        @source_id = source_id
+        @concepts = concepts
+      end
+
+      def resolve(reference)
+        return nil unless reference.is_a?(ConceptReference)
+        return nil unless reference.source == @source_id
+
+        concepts.by_id_and(reference.concept_id, reference.version)
+      end
+    end
+  end
+end

data/lib/glossarist/resolution_adapter.rb

@@ -4,6 +4,7 @@ module Glossarist
   class ResolutionAdapter
     autoload :Local, "glossarist/resolution_adapter/local"
     autoload :Package, "glossarist/resolution_adapter/package"
+    autoload :Bibliography, "glossarist/resolution_adapter/bibliography"
     autoload :Route, "glossarist/resolution_adapter/route"
     autoload :Remote, "glossarist/resolution_adapter/remote"
 

data/lib/glossarist/schema_migration/v0_to_v1.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class SchemaMigration
+    class V0ToV1
+      ENTRY_STATUS_MAP = {
+        "Standard" => "valid",
+        "Confirmed" => "valid",
+        "Proposed" => "draft",
+      }.freeze
+
+      LANG_CODES = Glossarist::LANG_CODES
+
+      IEV_PATTERN = /\{\{([^,}]+),\s*IEV:([^}]+)\}\}/
+      URN_PATTERN = /\{urn:iso:std:iso:(\d+):([^,}]+),([^}]+)\}/
+
+      attr_reader :from_version, :to_version
+
+      def initialize(concept_hash, from_version: "0",
+                               to_version: SchemaMigration::CURRENT_SCHEMA_VERSION,
+                               ref_maps: {})
+        @concept = concept_hash
+        @from_version = from_version
+        @to_version = to_version
+        @ref_maps = ref_maps
+      end
+
+      def migrate
+        case [from_version, to_version]
+        when ["0", "1"] then migrate_v0_to_v1
+        else
+          raise Errors::Base,
+                "Unsupported migration: #{from_version} -> #{to_version}"
+        end
+        @concept
+      end
+
+      private
+
+      def migrate_v0_to_v1
+        migrate_termid
+        LANG_CODES.each do |lang|
+          migrate_language_block(lang) if @concept[lang]
+        end
+        strip_revisions
+      end
+
+      def migrate_termid
+        if @concept.key?("termid")
+          @concept["termid"] =
+            String(@concept["termid"])
+        end
+      end
+
+      def migrate_language_block(lang)
+        l10n_block = @concept[lang]
+        return unless l10n_block.is_a?(Hash)
+
+        migrate_definition(l10n_block)
+        migrate_authoritative_source(l10n_block)
+        migrate_dates(l10n_block)
+        migrate_entry_status(l10n_block)
+        migrate_terms_abbrev(l10n_block)
+        extract_inline_refs(l10n_block)
+        strip_revisions(l10n_block)
+      end
+
+      def migrate_definition(l10n_block)
+        return unless l10n_block.key?("definition")
+        return unless l10n_block["definition"].is_a?(String)
+
+        l10n_block["definition"] = [{ "content" => l10n_block["definition"] }]
+      end
+
+      def migrate_authoritative_source(l10n_block)
+        return unless l10n_block.key?("authoritative_source")
+
+        src = l10n_block.delete("authoritative_source")
+        return if l10n_block.key?("sources")
+
+        sources = (src.is_a?(Array) ? src : [src]).filter_map do |s|
+          next unless s.is_a?(Hash)
+
+          origin = {}
+          origin["ref"] = s["ref"] if s["ref"]
+          origin["clause"] = s["clause"] if s["clause"]
+          origin["link"] = s["link"] if s["link"]
+
+          entry = { "type" => "authoritative", "origin" => origin }
+          if s["relationship"]
+            entry["status"] = s["relationship"]["type"] || "identical"
+            if s["relationship"]["modification"]
+              entry["modification"] =
+                s["relationship"]["modification"]
+            end
+          end
+          entry
+        end
+
+        l10n_block["sources"] = sources if sources.any?
+      end
+
+      def migrate_dates(l10n_block)
+        return if l10n_block.key?("dates")
+
+        dates = []
+        if l10n_block["date_accepted"]
+          dates << { "type" => "accepted",
+                     "date" => l10n_block["date_accepted"] }
+        end
+        if l10n_block["date_amended"]
+          dates << { "type" => "amended", "date" => l10n_block["date_amended"] }
+        end
+        l10n_block["dates"] = dates if dates.any?
+      end
+
+      def migrate_entry_status(l10n_block)
+        return unless l10n_block.key?("entry_status")
+
+        mapped = ENTRY_STATUS_MAP[l10n_block["entry_status"]]
+        l10n_block["entry_status"] = mapped if mapped
+      end
+
+      def migrate_terms_abbrev(l10n_block)
+        return unless l10n_block["terms"].is_a?(Array)
+
+        l10n_block["terms"].each do |term|
+          next unless term.is_a?(Hash)
+          next unless term["abbrev"] == true
+
+          term["type"] = "abbreviation"
+          term.delete("abbrev")
+        end
+      end
+
+      def extract_inline_refs(l10n_block)
+        texts = []
+
+        if l10n_block["definition"].is_a?(Array)
+          l10n_block["definition"].each do |d|
+            texts << (d.is_a?(Hash) ? d["content"].to_s : d.to_s)
+          end
+        elsif l10n_block["definition"].is_a?(String)
+          texts << l10n_block["definition"]
+        end
+
+        Array(l10n_block["notes"]).each do |n|
+          texts << (n.is_a?(Hash) ? n["content"].to_s : n.to_s)
+        end
+        Array(l10n_block["examples"]).each do |e|
+          texts << (e.is_a?(Hash) ? e["content"].to_s : e.to_s)
+        end
+
+        full_text = texts.join(" ")
+
+        refs = []
+
+        full_text.scan(IEV_PATTERN) do |term, id|
+          refs << {
+            "term" => term.strip,
+            "concept_id" => id.strip,
+            "source" => "urn:iec:std:iec:60050",
+            "ref_type" => "urn",
+          }
+        end
+
+        full_text.scan(URN_PATTERN) do |std_num, id, term|
+          refs << {
+            "term" => term.strip,
+            "concept_id" => id.strip,
+            "source" => "urn:iso:std:iso:#{std_num}",
+            "ref_type" => "urn",
+          }
+        end
+
+        return if refs.empty?
+
+        existing = l10n_block["references"] || []
+        seen_ids = existing.to_set { |r| r["concept_id"] || r["id"] }
+        refs.each do |ref|
+          key = ref["concept_id"] || ref["id"]
+          next if seen_ids.include?(key)
+
+          seen_ids.add(key)
+          existing << ref
+        end
+        l10n_block["references"] = existing
+      end
+
+      def strip_revisions(hash = @concept)
+        hash.delete("_revisions")
+        LANG_CODES.each do |lang|
+          next unless hash[lang].is_a?(Hash)
+
+          hash[lang].delete("_revisions")
+        end
+      end
+    end
+  end
+end