glossarist 2.6.1 → 2.6.3

data/lib/glossarist/gcr_package.rb

@@ -2,6 +2,7 @@
 
 require "zip"
 require "fileutils"
+require "pathname"
 
 module Glossarist
   class GcrPackage
@@ -14,12 +15,18 @@ module Glossarist
 
     KNOWN_COMPILED_FORMATS = COMPILED_EXTENSIONS.keys.freeze
 
-    attr_reader :zip_path, :metadata, :concepts
+    DATASET_ASSETS = [
+      { path: "bibliography.yaml", type: :file, attr: :bibliography },
+      { path: "images", type: :directory },
+    ].freeze
+
+    attr_reader :zip_path, :metadata, :concepts, :bibliography
 
     def initialize(zip_path)
       @zip_path = zip_path
       @metadata = nil
       @concepts = []
+      @bibliography = nil
     end
 
     def self.create(concepts:, metadata:, output_path:, register_data: nil,
@@ -66,12 +73,43 @@ module Glossarist
       end
     end
 
+    def self.each_dataset_asset(source_dir)
+      base = Pathname.new(source_dir)
+      DATASET_ASSETS.each do |asset|
+        path = File.join(source_dir, asset[:path])
+        case asset[:type]
+        when :file
+          yield_file_asset(path, asset[:path]) { |*a| yield(*a) }
+        when :directory
+          yield_directory_assets(path, base) { |*a| yield(*a) }
+        end
+      end
+    end
+
+    def self.yield_file_asset(path, entry_name)
+      return unless File.exist?(path)
+
+      yield entry_name, File.binread(path)
+    end
+
+    def self.yield_directory_assets(dir_path, base_path)
+      return unless File.directory?(dir_path)
+
+      Dir.glob(File.join(dir_path, "**", "*")).each do |file|
+        next unless File.file?(file)
+
+        relative = Pathname.new(file).relative_path_from(base_path).to_s
+        yield relative, File.binread(file)
+      end
+    end
+
     def validate
       GcrValidator.new.validate(@zip_path)
     end
 
-    def write(concepts, metadata, register_data, compiled_formats: [],
-              shortname: nil, **opts)
+    def write(concepts, metadata, register_data, # rubocop:disable Metrics/ParameterLists
+              compiled_formats: [],
+              shortname: nil, source_dir: nil, **opts)
       Zip::File.open(@zip_path, create: true) do |zf|
         zf.get_output_stream("metadata.yaml") do |f|
           f.write(metadata.to_yaml)
@@ -87,6 +125,12 @@ module Glossarist
           write_concept(zf, mc)
         end
 
+        if source_dir
+          self.class.each_dataset_asset(source_dir) do |name, content|
+            zf.get_output_stream(name) { |f| f.write(content) }
+          end
+        end
+
         if compiled_formats.any?
           write_compiled(zf, concepts, compiled_formats, shortname: shortname,
                                                          **opts)
@@ -94,29 +138,51 @@ module Glossarist
       end
     end
 
-    def write_concept(zip_file, concept)
-      termid = concept.data.id.to_s
-      doc = ConceptDocument.from_managed_concept(concept)
-      zip_file.get_output_stream("concepts/#{termid}.yaml") do |f|
-        f.write(doc.to_yamls)
-      end
-    end
-
     def read
       @concepts = []
 
       Zip::File.open(@zip_path) do |zf|
-        if (entry = zf.find_entry("metadata.yaml"))
-          @metadata = GcrMetadata.from_yaml(entry.get_input_stream.read)
-        end
+        read_metadata(zf)
+        read_file_assets(zf)
+        read_concepts(zf)
+      end
+    end
 
-        zf.entries.each do |entry|
-          next unless entry.name.start_with?("concepts/") && entry.name.end_with?(".yaml")
+    def read_metadata(zip_file)
+      entry = zip_file.find_entry("metadata.yaml")
+      return unless entry
 
-          raw = entry.get_input_stream.read
-          doc = ConceptDocument.from_yamls(raw)
-          @concepts << doc.to_managed_concept
-        end
+      @metadata = GcrMetadata.from_yaml(entry.get_input_stream.read)
+    end
+
+    def read_file_assets(zip_file)
+      DATASET_ASSETS.each do |asset|
+        next unless asset[:type] == :file && asset[:attr]
+
+        entry = zip_file.find_entry(asset[:path])
+        next unless entry
+
+        instance_variable_set("@#{asset[:attr]}", entry.get_input_stream.read)
+      end
+    end
+
+    def read_concepts(zip_file)
+      zip_file.entries.each do |entry|
+        next unless entry.name.start_with?("concepts/") && entry.name.end_with?(".yaml")
+
+        raw = entry.get_input_stream.read
+        doc = ConceptDocument.from_yamls(raw)
+        @concepts << doc.to_managed_concept
+      end
+    end
+
+    private
+
+    def write_concept(zip_file, concept)
+      termid = concept.data.id.to_s
+      doc = ConceptDocument.from_managed_concept(concept)
+      zip_file.get_output_stream("concepts/#{termid}.yaml") do |f|
+        f.write(doc.to_yamls)
       end
     end
 
@@ -206,6 +272,7 @@ compiled_formats: [], **opts)
           output_path: File.expand_path(output),
           compiled_formats: compiled_formats,
           shortname: shortname,
+          source_dir: dir,
           **opts,
         )
       end
@@ -219,7 +286,7 @@ compiled_formats: [], **opts)
         concept_count = 0
         languages = Set.new
 
-        Zip::OutputStream.open(output_path) do |zos|
+        Zip::OutputStream.open(output_path) do |zos| # rubocop:disable Metrics/BlockLength
           if register_data
             zos.put_next_entry("register.yaml")
             zos.write(register_data.to_yaml)
@@ -253,6 +320,11 @@ compiled_formats: [], **opts)
                                               register_data: register_data, **opts)
           zos.put_next_entry("metadata.yaml")
           zos.write(metadata.to_yaml)
+
+          each_dataset_asset(dir) do |name, content|
+            zos.put_next_entry(name)
+            zos.write(content)
+          end
         end
 
         new(output_path)

data/lib/glossarist/gcr_validator.rb

@@ -13,27 +13,8 @@ module Glossarist
       end
 
       begin
-        Zip::File.open(zip_path) do |zf|
-          unless zf.find_entry("metadata.yaml")
-            result.add_error("Missing metadata.yaml")
-            return result
-          end
-
-          metadata = GcrMetadata.from_yaml(
-            zf.find_entry("metadata.yaml").get_input_stream.read,
-          )
-          validate_metadata(metadata, result)
-
-          concept_entries = zf.entries.select do |e|
-            e.name.start_with?("concepts/") && e.name.end_with?(".yaml")
-          end
-          if concept_entries.empty?
-            result.add_error("No concept files found in concepts/")
-          end
-
-          concept_entries.each do |entry|
-            validate_concept_entry(entry, metadata, result)
-          end
+        Zip::File.open(zip_path) do |zip_file|
+          validate_zip_contents(zip_file, result)
         end
       rescue StandardError => e
         result.add_error("Failed to read ZIP: #{e.message}")
@@ -44,6 +25,31 @@ module Glossarist
 
     private
 
+    def validate_zip_contents(zip_file, result) # rubocop:disable Metrics/AbcSize
+      unless zip_file.find_entry("metadata.yaml")
+        result.add_error("Missing metadata.yaml")
+        return
+      end
+
+      metadata = GcrMetadata.from_yaml(
+        zip_file.find_entry("metadata.yaml").get_input_stream.read,
+      )
+      validate_metadata(metadata, result)
+
+      concept_entries = zip_file.entries.select do |e|
+        e.name.start_with?("concepts/") && e.name.end_with?(".yaml")
+      end
+      if concept_entries.empty?
+        result.add_error("No concept files found in concepts/")
+      end
+
+      concept_entries.each do |entry|
+        validate_concept_entry(entry, metadata, result)
+      end
+
+      validate_assets(zip_file, result)
+    end
+
     def validate_metadata(metadata, result)
       unless metadata&.concept_count
         result.add_error("metadata.yaml missing required fields (concept_count)")
@@ -94,5 +100,36 @@ module Glossarist
         result.add_warning("#{entry.name}: no concept URI (data.uri) and no concept_uri_template or uri_prefix in metadata")
       end
     end
+
+    def validate_assets(zip_file, result)
+      GcrPackage::DATASET_ASSETS.each do |asset|
+        case asset[:type]
+        when :file
+          validate_file_asset_entry(zip_file, asset[:path], result)
+        when :directory
+          validate_directory_asset(zip_file, asset[:path], result)
+        end
+      end
+    end
+
+    def validate_file_asset_entry(zip_file, path, result)
+      entry = zip_file.find_entry(path)
+      return unless entry
+
+      YAML.safe_load(entry.get_input_stream.read)
+    rescue Psych::SyntaxError => e
+      result.add_error("#{path}: invalid YAML at line #{e.line}: #{e.message}")
+    end
+
+    def validate_directory_asset(zip_file, dir_path, result)
+      dir_entries = zip_file.entries.select do |e|
+        e.name.start_with?("#{dir_path}/")
+      end
+      return unless dir_entries.any? && dir_entries.all? do |e|
+        e.name.end_with?("/")
+      end
+
+      result.add_warning("#{dir_path}/ directory is empty")
+    end
   end
 end

data/lib/glossarist/managed_concept.rb

@@ -1,4 +1,4 @@
-require_relative "localized_concept"
+# frozen_string_literal: true
 
 module Glossarist
   class ManagedConcept < Lutaml::Model::Serializable

data/lib/glossarist/rdf/skos_concept.rb

@@ -2,7 +2,6 @@
 
 require "lutaml/turtle"
 require "lutaml/jsonld"
-require_relative "../rdf"
 
 module Glossarist
   module Rdf

data/lib/glossarist/rdf/skos_vocabulary.rb

@@ -2,7 +2,6 @@
 
 require "lutaml/turtle"
 require "lutaml/jsonld"
-require_relative "../rdf"
 
 module Glossarist
   module Rdf

data/lib/glossarist/sts/extracted_designation.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Sts
+    ExtractedDesignation = Struct.new(
+      :term,
+      :type,
+      :normative_status,
+      :part_of_speech,
+      :abbreviation_type,
+      keyword_init: true,
+    )
+  end
+end

data/lib/glossarist/sts/extracted_lang_set.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Sts
+    ExtractedLangSet = Struct.new(
+      :language_code,
+      :definition_text,
+      :note_texts,
+      :example_texts,
+      :source_texts,
+      :domain,
+      :designations,
+      keyword_init: true,
+    )
+  end
+end

data/lib/glossarist/sts/extracted_term.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Sts
+    ExtractedTerm = Struct.new(
+      :id,
+      :label,
+      :source_ref,
+      :lang_sets,
+      keyword_init: true,
+    )
+  end
+end

data/lib/glossarist/sts/import_result.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Glossarist
+  module Sts
+    DuplicateConflict = Struct.new(:new_concept, :existing_concept, :key,
+                                   keyword_init: true)
+
+    class ImportResult
+      attr_reader :concepts, :conflicts, :source_files, :skipped_count
+
+      def initialize(concepts:, conflicts: [], source_files: [],
+skipped_count: 0)
+        @concepts = concepts
+        @conflicts = conflicts
+        @source_files = source_files
+        @skipped_count = skipped_count
+      end
+
+      def conflict?
+        !conflicts.empty?
+      end
+    end
+  end
+end

data/lib/glossarist/sts/importer.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require_relative "import_result"
+
+module Glossarist
+  module Sts
+    class Importer
+      STRATEGIES = %i[skip replace merge].freeze
+
+      attr_reader :duplicate_strategy
+
+      def initialize(duplicate_strategy: :skip)
+        unless STRATEGIES.include?(duplicate_strategy)
+          raise ArgumentError,
+                "duplicate_strategy must be one of #{STRATEGIES.join(', ')}, got #{duplicate_strategy}"
+        end
+
+        @duplicate_strategy = duplicate_strategy
+        @mapper = TermMapper.new
+      end
+
+      def import_new(xml_files, output:, shortname: nil, version: nil, **opts)
+        raw_concepts = extract_all_concepts(xml_files)
+        concepts, conflicts, skipped = dedup_concepts(raw_concepts)
+
+        if output.end_with?(".gcr")
+          unless shortname
+            raise ArgumentError,
+                  "--shortname is required for GCR output"
+          end
+          unless version
+            raise ArgumentError,
+                  "--version is required for GCR output"
+          end
+
+          create_gcr(concepts, output, shortname: shortname, version: version,
+                                       **opts)
+        else
+          save_dataset(concepts, output)
+        end
+
+        ImportResult.new(
+          concepts: concepts,
+          conflicts: conflicts,
+          source_files: xml_files,
+          skipped_count: skipped,
+        )
+      end
+
+      def import_into_existing(xml_files, dataset_path)
+        existing = load_existing(dataset_path)
+        new_concepts = extract_all_concepts(xml_files)
+        index = build_concept_index(existing)
+
+        result_state = apply_with_dedup(new_concepts, existing, index)
+
+        save_to_path(existing, dataset_path)
+
+        ImportResult.new(
+          concepts: existing.managed_concepts,
+          conflicts: result_state.conflicts,
+          source_files: xml_files,
+          skipped_count: result_state.skipped,
+        )
+      end
+
+      DedupState = Struct.new(:conflicts, :skipped, keyword_init: true)
+
+      private
+
+      def apply_with_dedup(new_concepts, existing, index)
+        state = DedupState.new(conflicts: [], skipped: 0)
+
+        new_concepts.each do |mc|
+          key = concept_key(mc)
+          existing_mc = index[key]
+
+          if existing_mc.nil?
+            existing.store(mc)
+            index[key] = mc
+          else
+            state.conflicts << DuplicateConflict.new(
+              new_concept: mc, existing_concept: existing_mc, key: key,
+            )
+            handle_duplicate(existing, existing_mc, mc, index, key, state)
+          end
+        end
+
+        state
+      end
+
+      def handle_duplicate(existing, old_mc, new_mc, index, key, state)
+        case duplicate_strategy
+        when :skip
+          state.skipped += 1
+        when :replace
+          replace_in_collection(existing, old_mc, new_mc)
+          index[key] = new_mc
+        when :merge
+          merge_concept(old_mc, new_mc)
+        end
+      end
+
+      def extract_all_concepts(xml_files)
+        xml_files.flat_map do |path|
+          extractor = TermExtractor.new(path)
+          terms = extractor.extract
+          terms.map { |t| @mapper.map(t) }
+        end
+      end
+
+      def dedup_concepts(concepts) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+        seen = {}
+        conflicts = []
+        skipped = 0
+        unique = []
+
+        concepts.each do |mc|
+          key = concept_key(mc)
+          if key.first.empty? || seen[key].nil?
+            unique << mc
+            seen[key] = mc unless key.first.empty?
+          else
+            conflicts << DuplicateConflict.new(
+              new_concept: mc, existing_concept: seen[key], key: key,
+            )
+            skipped += apply_dedup_to_unique(unique, seen, mc, key)
+          end
+        end
+
+        [unique, conflicts, skipped]
+      end
+
+      def apply_dedup_to_unique(unique, seen, new_mc, key)
+        case duplicate_strategy
+        when :skip
+          1
+        when :replace
+          unique.delete(seen[key])
+          unique << new_mc
+          seen[key] = new_mc
+          0
+        when :merge
+          merge_concept(seen[key], new_mc)
+          0
+        end
+      end
+
+      def concept_key(managed_concept)
+        designation = managed_concept.default_designation.to_s.downcase.strip
+        domain = begin
+          l10n = managed_concept.default_lang
+          l10n&.data&.domain.to_s.downcase.strip
+        end
+        [designation, domain]
+      end
+
+      def build_concept_index(collection)
+        index = {}
+        collection.each do |mc|
+          key = concept_key(mc)
+          index[key] = mc unless key.first.empty?
+        end
+        index
+      end
+
+      def merge_concept(existing_mc, new_mc)
+        new_mc.localizations.each do |l10n|
+          lang = l10n.language_code
+          if existing_mc.localization(lang).nil?
+            existing_mc.add_localization(l10n)
+          end
+        end
+      end
+
+      def replace_in_collection(collection, old_mc, new_mc)
+        collection.managed_concepts.delete(old_mc)
+        collection.store(new_mc)
+      end
+
+      def load_existing(path)
+        collection = ManagedConceptCollection.new
+        if path.end_with?(".gcr")
+          package = GcrPackage.load(path)
+          package.concepts.each { |mc| collection.store(mc) }
+        else
+          concepts = ConceptCollector.collect(path)
+          concepts.each { |mc| collection.store(mc) }
+        end
+        collection
+      end
+
+      def save_to_path(collection, path)
+        if path.end_with?(".gcr")
+          tmpdir = build_temp_dataset(collection.managed_concepts)
+          begin
+            GC.start
+            tmp_gcr = "#{path}.tmp.#{Process.pid}"
+            GcrPackage.create_from_directory(
+              tmpdir,
+              output: tmp_gcr,
+              shortname: File.basename(path, ".gcr"),
+              version: "1.0.0",
+            )
+            FileUtils.rm_f(path)
+            FileUtils.mv(tmp_gcr, path)
+          ensure
+            FileUtils.rm_rf(tmpdir)
+            FileUtils.rm_f(tmp_gcr)
+          end
+        else
+          save_dataset(collection.managed_concepts, path)
+        end
+      end
+
+      def save_dataset(concepts, dir)
+        concepts_dir = File.join(dir, "concepts")
+        FileUtils.mkdir_p(concepts_dir)
+        collection = ManagedConceptCollection.new
+        concepts.each { |mc| collection.store(mc) }
+        collection.save_grouped_concepts_to_files(concepts_dir)
+      end
+
+      def create_gcr(concepts, output, shortname:, version:, **opts)
+        tmpdir = build_temp_dataset(concepts)
+        begin
+          GcrPackage.create_from_directory(
+            tmpdir,
+            output: output,
+            shortname: shortname,
+            version: version,
+            **opts,
+          )
+        ensure
+          FileUtils.rm_rf(tmpdir)
+        end
+      end
+
+      def build_temp_dataset(concepts)
+        tmpdir = Dir.mktmpdir("glossarist-sts-import")
+        concepts_dir = File.join(tmpdir, "concepts")
+        FileUtils.mkdir_p(concepts_dir)
+
+        collection = ManagedConceptCollection.new
+        concepts.each { |mc| collection.store(mc) }
+        collection.save_grouped_concepts_to_files(concepts_dir)
+
+        tmpdir
+      end
+    end
+  end
+end