glossarist 2.8.7 → 2.8.10

data/lib/glossarist/schema_migration/v2_to_v3.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Glossarist
+  class SchemaMigration
+    module V2ToV3
+      def self.migrate_concept(concept, target_version: Glossarist::SCHEMA_VERSION)
+        current = concept_version(concept)
+        target = target_version.to_s
+
+        return concept if current == target
+
+        max_steps = 5
+        max_steps.times do
+          break if current == target
+
+          case current
+          when "2" then current = step_v2_to_v3(concept)
+          else
+            raise Errors::Base,
+                  "No concept migration step from version #{current}"
+          end
+        end
+
+        unless current == target
+          raise Errors::Base,
+                "Migration chain too long or unresolvable"
+        end
+
+        concept.schema_version = target
+        concept
+      end
+
+      def self.concept_version(concept)
+        version = concept.schema_version
+        return version.to_s if version && !version.to_s.empty?
+
+        ManagedConcept.detect_schema_version(concept)
+      end
+
+      def self.step_v2_to_v3(concept)
+        if concept.data&.related&.any?
+          concept.related ||= []
+          concept.related = (concept.related + concept.data.related).uniq
+          concept.data.related = []
+        end
+        "3"
+      end
+    end
+  end
+end

data/lib/glossarist/schema_migration.rb

@@ -6,76 +6,19 @@ module Glossarist
   class SchemaMigration
     CURRENT_SCHEMA_VERSION = "1"
 
-    def self.migrate_concept(concept, target_version: Glossarist::SCHEMA_VERSION)
-      current = concept_version(concept)
-      target = target_version.to_s
-
-      return concept if current == target
-
-      max_steps = 5
-      max_steps.times do
-        break if current == target
-
-        case current
-        when "2" then current = step_v2_to_v3(concept)
-        else
-          raise Error, "No concept migration step from version #{current}"
-        end
-      end
-
-      unless current == target
-        raise Error,
-              "Migration chain too long or unresolvable"
-      end
-
-      concept.schema_version = target
-      concept
-    end
-
-    def self.concept_version(concept)
-      version = concept.schema_version
-      return version.to_s if version && !version.to_s.empty?
-
-      ManagedConcept.detect_schema_version(concept)
-    end
+    autoload :V0ToV1, "glossarist/schema_migration/v0_to_v1"
+    autoload :V2ToV3, "glossarist/schema_migration/v2_to_v3"
 
-    def self.step_v2_to_v3(concept)
-      if concept.data&.related&.any?
-        concept.related ||= []
-        concept.related = (concept.related + concept.data.related).uniq
-        concept.data.related = []
-      end
-      "3"
+    def self.new(...)
+      V0ToV1.new(...)
     end
 
-    ENTRY_STATUS_MAP = {
-      "Standard" => "valid",
-      "Confirmed" => "valid",
-      "Proposed" => "draft",
-    }.freeze
-
-    LANG_CODES = Glossarist::LANG_CODES
-
-    IEV_PATTERN = /\{\{([^,}]+),\s*IEV:([^}]+)\}\}/
-    URN_PATTERN = /\{urn:iso:std:iso:(\d+):([^,}]+),([^}]+)\}/
-
-    attr_reader :from_version, :to_version
-
-    def initialize(concept_hash, from_version: "0",
-                             to_version: CURRENT_SCHEMA_VERSION, ref_maps: {})
-      @concept = concept_hash
-      @from_version = from_version
-      @to_version = to_version
-      @ref_maps = ref_maps
+    def self.migrate_concept(concept, target_version: Glossarist::SCHEMA_VERSION)
+      V2ToV3.migrate_concept(concept, target_version: target_version)
     end
 
-    def migrate
-      case [from_version, to_version]
-      when ["0", "1"] then migrate_v0_to_v1
-      else
-        raise Error, "Unsupported migration: #{from_version} -> #{to_version}"
-      end
-      @concept
+    def self.concept_version(concept)
+      V2ToV3.concept_version(concept)
     end
 
     def self.upgrade_directory(source_dir, output:, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
@@ -112,163 +55,6 @@ module Glossarist
       }
     end
 
-    private
-
-    def migrate_v0_to_v1
-      migrate_termid
-      LANG_CODES.each do |lang|
-        migrate_language_block(lang) if @concept[lang]
-      end
-      strip_revisions
-    end
-
-    def migrate_termid
-      @concept["termid"] = String(@concept["termid"]) if @concept.key?("termid")
-    end
-
-    def migrate_language_block(lang)
-      lc = @concept[lang]
-      return unless lc.is_a?(Hash)
-
-      migrate_definition(lc)
-      migrate_authoritative_source(lc)
-      migrate_dates(lc)
-      migrate_entry_status(lc)
-      migrate_terms_abbrev(lc)
-      extract_inline_refs(lc)
-      strip_revisions(lc)
-    end
-
-    def migrate_definition(lc)
-      return unless lc.key?("definition")
-      return unless lc["definition"].is_a?(String)
-
-      lc["definition"] = [{ "content" => lc["definition"] }]
-    end
-
-    def migrate_authoritative_source(lc)
-      return unless lc.key?("authoritative_source")
-
-      src = lc.delete("authoritative_source")
-      return if lc.key?("sources")
-
-      sources = (src.is_a?(Array) ? src : [src]).filter_map do |s|
-        next unless s.is_a?(Hash)
-
-        origin = {}
-        origin["ref"] = s["ref"] if s["ref"]
-        origin["clause"] = s["clause"] if s["clause"]
-        origin["link"] = s["link"] if s["link"]
-
-        entry = { "type" => "authoritative", "origin" => origin }
-        if s["relationship"]
-          entry["status"] = s["relationship"]["type"] || "identical"
-          if s["relationship"]["modification"]
-            entry["modification"] =
-              s["relationship"]["modification"]
-          end
-        end
-        entry
-      end
-
-      lc["sources"] = sources if sources.any?
-    end
-
-    def migrate_dates(lc)
-      return if lc.key?("dates")
-
-      dates = []
-      if lc["date_accepted"]
-        dates << { "type" => "accepted", "date" => lc["date_accepted"] }
-      end
-      if lc["date_amended"]
-        dates << { "type" => "amended", "date" => lc["date_amended"] }
-      end
-      lc["dates"] = dates if dates.any?
-    end
-
-    def migrate_entry_status(lc)
-      return unless lc.key?("entry_status")
-
-      mapped = ENTRY_STATUS_MAP[lc["entry_status"]]
-      lc["entry_status"] = mapped if mapped
-    end
-
-    def migrate_terms_abbrev(lc)
-      return unless lc["terms"].is_a?(Array)
-
-      lc["terms"].each do |term|
-        next unless term.is_a?(Hash)
-        next unless term["abbrev"] == true
-
-        term["type"] = "abbreviation"
-        term.delete("abbrev")
-      end
-    end
-
-    def extract_inline_refs(lc)
-      texts = []
-
-      if lc["definition"].is_a?(Array)
-        lc["definition"].each do |d|
-          texts << (d.is_a?(Hash) ? d["content"].to_s : d.to_s)
-        end
-      elsif lc["definition"].is_a?(String)
-        texts << lc["definition"]
-      end
-
-      Array(lc["notes"]).each do |n|
-        texts << (n.is_a?(Hash) ? n["content"].to_s : n.to_s)
-      end
-      Array(lc["examples"]).each do |e|
-        texts << (e.is_a?(Hash) ? e["content"].to_s : e.to_s)
-      end
-
-      full_text = texts.join(" ")
-
-      refs = []
-
-      full_text.scan(IEV_PATTERN) do |term, id|
-        refs << {
-          "term" => term.strip,
-          "concept_id" => id.strip,
-          "source" => "urn:iec:std:iec:60050",
-          "ref_type" => "urn",
-        }
-      end
-
-      full_text.scan(URN_PATTERN) do |std_num, id, term|
-        refs << {
-          "term" => term.strip,
-          "concept_id" => id.strip,
-          "source" => "urn:iso:std:iso:#{std_num}",
-          "ref_type" => "urn",
-        }
-      end
-
-      return if refs.empty?
-
-      existing = lc["references"] || []
-      seen_ids = existing.to_set { |r| r["concept_id"] || r["id"] }
-      refs.each do |ref|
-        key = ref["concept_id"] || ref["id"]
-        next if seen_ids.include?(key)
-
-        seen_ids.add(key)
-        existing << ref
-      end
-      lc["references"] = existing
-    end
-
-    def strip_revisions(hash = @concept)
-      hash.delete("_revisions")
-      LANG_CODES.each do |lang|
-        next unless hash[lang].is_a?(Hash)
-
-        hash[lang].delete("_revisions")
-      end
-    end
-
     class << self
       private
 
@@ -301,14 +87,14 @@ module Glossarist
           v1 = V1::Concept.from_file(file)
           next unless v1
 
-          migration = new(
+          migration = V0ToV1.new(
             v1.to_yaml_hash,
             from_version: source_version,
             to_version: target_version,
             ref_maps: ref_maps,
           )
           concepts << migration.migrate
-        rescue StandardError => e
+        rescue Errors::Base, Psych::SyntaxError => e
           errors += 1
           warn "  Error migrating #{File.basename(file)}: #{e.message}" if errors <= 5
         end

data/lib/glossarist/sts/importer.rb

@@ -184,8 +184,9 @@ module Glossarist
           package = GcrPackage.load(path)
           package.concepts.each { |mc| collection.store(mc) }
         else
-          concepts = ConceptCollector.collect(path)
-          concepts.each { |mc| collection.store(mc) }
+          GlossaryStore.new.tap do |s|
+            s.load(path)
+          end.each_concept { |mc| collection.store(mc) }
         end
         collection
       end
@@ -214,11 +215,10 @@ module Glossarist
       end
 
       def save_dataset(concepts, dir)
-        concepts_dir = File.join(dir, "concepts")
-        FileUtils.mkdir_p(concepts_dir)
-        collection = ManagedConceptCollection.new
-        concepts.each { |mc| collection.store(mc) }
-        collection.save_grouped_concepts_to_files(concepts_dir)
+        FileUtils.mkdir_p(dir)
+        store = GlossaryStore.new
+        concepts.each { |mc| store.add_concept(mc) }
+        store.save_directory(dir)
       end
 
       def create_gcr(concepts, output, shortname:, version:, **opts)
@@ -238,12 +238,11 @@ module Glossarist
 
       def build_temp_dataset(concepts)
         tmpdir = Dir.mktmpdir("glossarist-sts-import")
-        concepts_dir = File.join(tmpdir, "concepts")
-        FileUtils.mkdir_p(concepts_dir)
+        FileUtils.mkdir_p(tmpdir)
 
-        collection = ManagedConceptCollection.new
-        concepts.each { |mc| collection.store(mc) }
-        collection.save_grouped_concepts_to_files(concepts_dir)
+        store = GlossaryStore.new
+        concepts.each { |mc| store.add_concept(mc) }
+        store.save_directory(tmpdir)
 
         tmpdir
       end

data/lib/glossarist/sts/term_extractor.rb

@@ -7,6 +7,7 @@ module Glossarist
         raw = File.read(xml_path)
         @standard = ::Sts::IsoSts::Standard.from_xml(raw)
         @source_ref = extract_source_ref
+        @std_prefix = extract_std_prefix(@source_ref)
       end
 
       def extract
@@ -90,27 +91,104 @@ module Glossarist
         )
       end
 
+      ELEMENT_NAME_TO_ATTR = {
+        "entailedTerm" => :entailed_term,
+        "xref" => :xref,
+        "italic" => :italic,
+        "bold" => :bold,
+        "sup" => :sup,
+        "sub" => :sub,
+        "monospace" => :monospace,
+        "std" => :std,
+        "math" => :math,
+        "inline-formula" => :inline_formula,
+        "list" => :list,
+        "styled-content" => :styled_content,
+        "ext-link" => :ext_link,
+      }.freeze
+
       def extract_definition_text(lang_set)
         definitions = lang_set.definition
         return "" unless definitions&.any?
 
-        definitions.first.value&.join.to_s.strip
+        extract_mixed_text(definitions.first)
       end
 
       def extract_note_texts(lang_set)
         lang_set.note.filter_map do |n|
-          text = n.value&.join.to_s.strip
+          text = extract_mixed_text(n)
           text unless text.empty?
         end
       end
 
       def extract_example_texts(lang_set)
         lang_set.example.filter_map do |e|
-          text = e.value&.join.to_s.strip
+          text = extract_mixed_text(e)
           text unless text.empty?
         end
       end
 
+      def extract_mixed_text(mixed_element)
+        indices = Hash.new(0)
+        parts = []
+
+        mixed_element.element_order.each do |elem|
+          if elem.node_type == :text
+            parts << elem.text_content.to_s
+          else
+            attr_name = ELEMENT_NAME_TO_ATTR[elem.name]
+            next unless attr_name
+
+            collection = mixed_element.class.attributes.key?(attr_name) &&
+              mixed_element.public_send(attr_name)
+            next unless collection
+
+            child = collection[indices[elem.name]]
+            if child
+              if elem.name == "entailedTerm"
+                parts << format_entailed_term(child)
+              else
+                text = child_value_text(child)
+                parts << text if text
+              end
+            end
+            indices[elem.name] += 1
+          end
+        end
+
+        normalize_whitespace(parts.join)
+      end
+
+      # entailedTerm → "{{19135:2026:3.5.1,concept}}"
+      # format: {{concept_id, render_text}}
+      def format_entailed_term(entailed)
+        raw_text = entailed.value.to_s
+        designation = raw_text.gsub(/\s+\(\d[\d.]*\)\s*\z/, "").strip
+        section = extract_section_from_target(entailed.target)
+
+        if @std_prefix && section
+          "{{#{@std_prefix}:#{section},#{designation}}}"
+        else
+          raw_text.strip
+        end
+      end
+
+      # "term_3.5.1" → "3.5.1", "term_3.8.2-1" → "3.8.2"
+      def extract_section_from_target(target)
+        return nil unless target
+
+        match = target.match(/term_(\d+(?:\.\d+)*)/)
+        match ? match[1] : nil
+      end
+
+      def child_value_text(child)
+        val = child.value
+        case val
+        when Array then val.join.to_s
+        when String then val
+        end
+      end
+
       def extract_source_texts(lang_set)
         lang_set.source.filter_map do |s|
           text = s.value&.join.to_s.strip
@@ -175,12 +253,32 @@ module Glossarist
       end
 
       def extract_ref_text(ref)
-        if ref.value.is_a?(String)
-          ref.value.to_s.strip
+        if ref.respond_to?(:content) && ref.content.is_a?(Array)
+          normalize_whitespace(ref.content.join.to_s)
+        elsif ref.respond_to?(:value)
+          normalize_whitespace(ref.value.to_s)
         else
-          ref.content&.join.to_s.strip
+          ""
         end
       end
+
+      def normalize_whitespace(text)
+        text.gsub(/[\s\u00a0]+/, " ").strip
+      end
+
+      # "ISO 19101-1:2014" → "19101-1:2014", "ISO/TS 19130-2:2014" → "TS-19130-2:2014"
+      def extract_std_prefix(source_ref)
+        return nil unless source_ref
+
+        match = source_ref.match(/\AISO(?:\/(\p{Upper}+))? (\d+(?:-\d+)?):(\d+)\z/)
+        return nil unless match
+
+        type_part = match[1]
+        number = match[2]
+        year = match[3]
+
+        type_part ? "#{type_part}-#{number}:#{year}" : "#{number}:#{year}"
+      end
     end
   end
 end

data/lib/glossarist/validation/asset_index.rb

@@ -28,7 +28,7 @@ module Glossarist
       end
 
       def self.build_from_directory(dataset_path)
-        concepts = ConceptCollector.collect(dataset_path)
+        concepts = GlossaryStore.new.tap { |s| s.load(dataset_path) }.concepts
         build_from_concepts(concepts, dataset_path)
       end
 

data/lib/glossarist/validation/rules/cite_ref_integrity_rule.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Validation
+    module Rules
+      class CiteRefIntegrityRule < Base
+        def code = "GLS-110"
+        def category = :references
+        def severity = "warning"
+        def scope = :concept
+
+        def applicable?(context)
+          context.concept.localizations&.any?
+        end
+
+        def check(context)
+          concept = context.concept
+          fname = context.file_name
+          issues = []
+
+          check_unique_source_ids(concept, fname, issues)
+          check_unresolved_mentions(concept, fname, issues)
+
+          issues
+        end
+
+        private
+
+        def check_unique_source_ids(concept, fname, issues)
+          seen = Hash.new { |h, k| h[k] = [] }
+          concept.all_sources.each do |source|
+            next unless source.id
+
+            seen[source.id] << source
+          end
+
+          seen.each do |id, sources|
+            next if sources.length <= 1
+
+            issues << issue(
+              "duplicate source id '#{id}' appears #{sources.length} times",
+              code: "GLS-110", severity: severity,
+              location: fname,
+              suggestion: "source ids must be unique within a concept"
+            )
+          end
+        end
+
+        def check_unresolved_mentions(concept, fname, issues)
+          keys = cite_mention_keys(concept)
+          return if keys.empty?
+
+          known_ids = concept.all_sources.filter_map(&:id).to_set
+          keys.each do |key|
+            next if known_ids.include?(key)
+
+            issues << issue(
+              "inline {{cite:#{key}}} does not resolve to any source",
+              code: "GLS-110", severity: severity,
+              location: fname,
+              suggestion: "add a source with id '#{key}' or fix the reference"
+            )
+          end
+        end
+
+        def cite_mention_keys(concept)
+          extractor = ReferenceExtractor.new
+          extractor.extract_from_managed_concept(concept)
+            .select(&:cite?)
+            .filter_map(&:concept_id)
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/version.rb

@@ -4,5 +4,5 @@
 #
 
 module Glossarist
-  VERSION = "2.8.7"
+  VERSION = "2.8.10"
 end

data/lib/glossarist.rb

@@ -27,35 +27,27 @@ module Glossarist
   autoload :ReferenceResolver,        "glossarist/reference_resolver"
   autoload :ResolutionAdapter,        "glossarist/resolution_adapter"
   autoload :ConceptDate,              "glossarist/concept_date"
-  autoload :ConceptManager,           "glossarist/concept_manager"
   autoload :ConceptSet,               "glossarist/concept_set"
   autoload :ConceptSource,            "glossarist/concept_source"
   autoload :ConceptStore,             "glossarist/concept_store"
   autoload :ConceptValidator,         "glossarist/concept_validator"
-  autoload :ConceptCollector,         "glossarist/concept_collector"
   autoload :ConceptComparator,        "glossarist/concept_comparator"
-  autoload :ContextConfiguration, "glossarist/context_configuration"
+  autoload :ContextConfiguration,     "glossarist/context_configuration"
   autoload :ComparisonResult,         "glossarist/comparison_result"
   autoload :ConceptDiff,              "glossarist/concept_diff"
   autoload :ConceptDocument,          "glossarist/concept_document"
-  autoload :ConceptEnricher,          "glossarist/concept_enricher"
+  autoload :ConceptEnricher, "glossarist/concept_enricher"
   autoload :Config,                   "glossarist/config"
   autoload :DatasetValidator,         "glossarist/dataset_validator"
   autoload :CustomLocality,           "glossarist/custom_locality"
   autoload :DetailedDefinition,       "glossarist/detailed_definition"
   autoload :Designation,              "glossarist/designation"
-  autoload :Error,                    "glossarist/error"
+  autoload :Errors,                   "glossarist/errors"
   autoload :GcrPackage,               "glossarist/gcr_package"
   autoload :GcrPackageDefinition,     "glossarist/gcr_package_definition"
   autoload :GcrMetadata,              "glossarist/gcr_metadata"
   autoload :GcrStatistics,            "glossarist/gcr_statistics"
   autoload :GcrValidator,             "glossarist/gcr_validator"
-  autoload :InvalidTypeError,         "glossarist/error/invalid_type_error"
-  autoload :InvalidLanguageCodeError,
-           "glossarist/error/invalid_language_code_error"
-  autoload :ParseError, "glossarist/error/parse_error"
-  autoload :CacheVersionMismatchError,
-           "glossarist/error/cache_version_mismatch_error"
   autoload :Locality,                 "glossarist/locality"
   autoload :LocalizedConcept,         "glossarist/localized_concept"
   autoload :ManagedConcept,           "glossarist/managed_concept"
@@ -72,8 +64,8 @@ module Glossarist
   autoload :Utilities,                "glossarist/utilities"
   autoload :Validation,               "glossarist/validation"
   autoload :RegisterData,             "glossarist/register_data"
-  autoload :Section,                 "glossarist/section"
-  autoload :DatasetRegister,         "glossarist/dataset_register"
+  autoload :Section,                  "glossarist/section"
+  autoload :DatasetRegister,          "glossarist/dataset_register"
   autoload :ValidationResult,         "glossarist/validation_result"
   autoload :V1,                       "glossarist/v1"
   autoload :V2,                       "glossarist/v2"

data/scripts/upgrade_dataset_to_v3.rb

@@ -32,7 +32,7 @@ manager.load_from_files(collection: collection)
 
 count = 0
 collection.each do |concept|
-  Glossarist::SchemaMigration.migrate_concept(concept)
+  Glossarist::SchemaMigration::V2ToV3.migrate_concept(concept)
   count += 1
 end