suma 0.5.9 → 0.5.10

data/lib/suma/translation_ingester.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "glossarist"
+require "yaml"
+
+module Suma
+  # Overlays translated content onto extracted Glossarist concepts (T5).
+  #
+  # A translations file is a YAML sequence of localized-concept documents,
+  # each carrying an identity key plus the content to apply:
+  #
+  #   - concept_id: example_schema.bolt   # managed id (data.identifier)
+  #                                       # or the managed concept's uuid
+  #   - id: <localized uuid>              # alternative identity key
+  #     data:
+  #       language_code: fra
+  #       terms: [...]
+  #       definition: [...]
+  #
+  # Every field the entry provides replaces the localization's field; every
+  # field it omits falls back, explicitly, to the localization's current
+  # content (for a new language: the source-language localization's). All
+  # (de)serialization goes through the Glossarist/lutaml-model framework;
+  # entries are merged as serialized hashes and reified with of_yaml.
+  class TranslationIngester
+    def initialize(concepts_dir:, translations:, output_dir: nil)
+      @concepts_dir = concepts_dir
+      @translations = translations
+      @output_dir = output_dir || concepts_dir
+    end
+
+    # @return [Hash{String=>Array<String>}] concept file → applied entry
+    #   identities
+    def ingest
+      entries = load_entries
+      applied = {}
+      each_concept_document do |file, doc|
+        matched = entries_for(doc, entries)
+        next if matched.empty?
+
+        applied[file] = matched.map { |entry| apply_entry(doc, entry) }
+        write_document(file, doc)
+      end
+      assert_no_unmatched!(entries, applied)
+      applied
+    end
+
+    private
+
+    def load_entries
+      content = YAML.safe_load_file(@translations, permitted_classes: [],
+                                                   aliases: true)
+      entries = content.is_a?(Hash) ? content["translations"] : content
+      unless entries.is_a?(Array)
+        raise Error,
+              "translations file must be a YAML sequence of localized " \
+              "concepts: #{@translations}"
+      end
+
+      entries
+    end
+
+    def each_concept_document
+      Dir.glob(File.join(@concepts_dir, "*.yaml")).each do |file|
+        doc = Glossarist::V3::ConceptDocument.from_yamls(File.read(file))
+        yield file, doc
+      end
+    end
+
+    def entries_for(doc, entries)
+      entries.select { |entry| entry_matches?(doc, entry) }
+    end
+
+    def entry_matches?(doc, entry)
+      if entry["concept_id"]
+        [doc.concept.data.id, doc.concept.uuid].include?(entry["concept_id"])
+      else
+        localized_uuids(doc).include?(entry["id"])
+      end
+    end
+
+    def localized_uuids(doc)
+      doc.localizations.map(&:id) +
+        doc.concept.data.localized_concepts.values
+    end
+
+    def apply_entry(doc, entry)
+      incoming = Glossarist::LocalizedConcept
+        .of_yaml(entry.except("concept_id"))
+      localized = merge_localization(doc, entry, incoming)
+      replace_localization(doc, localized)
+      entry["concept_id"] || entry["id"]
+    end
+
+    def replace_localization(doc, localized)
+      language = localized.data.language_code
+      doc.localizations.reject! { |l10n| l10n.data.language_code == language }
+      doc.localizations << localized
+      doc.concept.data.localized_concepts[language] = localized.uuid
+    end
+
+    # Entry fields win; absent fields keep the localization's current
+    # content (or the source-language content for a new language). The
+    # merge happens on the raw entry data — a parsed model would
+    # materialize empty collections and wipe the fields it omits.
+    #
+    # Glossarist parses the serialized "id" key into `#id` (not `#uuid`),
+    # so the persisted id is preserved and stamped back onto the merged
+    # localization explicitly; a new language gets the same v5 id
+    # TermExtractor's linked localizations use.
+    def merge_localization(doc, entry, incoming)
+      current = current_localization(doc, entry, incoming)
+      base = current || doc.localizations.first
+      return incoming unless base
+
+      identity = entry["id"] || current&.id ||
+        new_localization_uuid(doc, incoming.data.language_code)
+      localized = Glossarist::LocalizedConcept
+        .of_yaml(merged_hash(base, entry, identity))
+      localized.uuid = identity
+      localized
+    end
+
+    def merged_hash(base, entry, identity)
+      merged = base.to_yaml_hash
+      merged["data"] = (merged["data"] || {})
+        .merge(entry["data"] || {})
+      merged["id"] = identity
+      merged
+    end
+
+    def current_localization(doc, entry, incoming)
+      if entry["id"]
+        doc.localizations.find { |l10n| l10n.id == entry["id"] }
+      else
+        doc.localizations.find do |l10n|
+          l10n.data.language_code == incoming.data.language_code
+        end
+      end
+    end
+
+    # Same convention as TermExtractor's linked localizations.
+    def new_localization_uuid(doc, language)
+      Glossarist::Utilities::UUID.uuid_v5(
+        Glossarist::Utilities::UUID::OID_NAMESPACE,
+        "#{doc.concept.data.id}-#{language}",
+      )
+    end
+
+    def write_document(file, doc)
+      output = File.join(@output_dir, File.basename(file))
+      FileUtils.mkdir_p(@output_dir) if output != file
+      File.write(output, doc.to_yamls, encoding: "utf-8")
+    end
+
+    def assert_no_unmatched!(entries, applied)
+      matched = applied.values.flatten
+      unmatched = entries.reject do |entry|
+        matched.include?(entry["concept_id"] || entry["id"])
+      end
+      return if unmatched.empty?
+
+      raise Error,
+            "unmatched translation entries: " \
+            "#{unmatched.map { |e| e['concept_id'] || e['id'] }.join(', ')}"
+    end
+  end
+end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.5.9"
+  VERSION = "0.5.10"
 end

data/lib/suma.rb

@@ -7,6 +7,8 @@ require "fileutils"
 module Suma
   autoload :VERSION, "suma/version"
 
+  autoload :AnchorValidator,        "suma/anchor_validator"
+  autoload :ArtifactServer,         "suma/artifact_server"
   autoload :SchemaProfile,          "suma/schema_profile"
   autoload :BuildPlan,              "suma/build_plan"
   autoload :Processor,              "suma/processor"
@@ -20,14 +22,17 @@ module Suma
   autoload :CorpusGraph,            "suma/corpus_graph"
   autoload :CollectionManifest,     "suma/collection_manifest"
   autoload :EengineConverter,       "suma/eengine_converter"
+  autoload :GcrPackage,             "suma/gcr_package"
   autoload :ExpressReformatter,     "suma/express_reformatter"
   autoload :ExpressSchema,          "suma/express_schema"
   autoload :LinkValidation,         "suma/link_validation"
+  autoload :LinkValidationResult,   "suma/link_validation_result"
   autoload :LinkValidator,          "suma/link_validator"
   autoload :ManifestTraverser,      "suma/manifest_traverser"
   autoload :MappingDrift,           "suma/mapping_drift"
   autoload :NoteProcessor,          "suma/note_processor"
   autoload :NullCache,              "suma/null_cache"
+  autoload :OutputCanonicalizer,    "suma/output_canonicalizer"
   autoload :OwlProjection,          "suma/owl_projection"
   autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
   autoload :SchemaCache,            "suma/schema_cache"
@@ -42,11 +47,13 @@ module Suma
   autoload :SchemaNaming,           "suma/schema_naming"
   autoload :SchemaTemplate,         "suma/schema_template"
   autoload :SiteConfig,             "suma/site_config"
+  autoload :SiteBuild,              "suma/site_build"
   autoload :StagedCollectionBuilder, "suma/staged_collection_builder"
   autoload :SvgQuality,             "suma/svg_quality"
   autoload :TermClassification,     "suma/term_classification"
   autoload :TermExtractor,          "suma/term_extractor"
   autoload :ThorExt,                "suma/thor_ext"
+  autoload :TranslationIngester,    "suma/translation_ingester"
   autoload :Urn,                    "suma/urn"
   autoload :UrnResolver,            "suma/urn_resolver"
   autoload :Utils,                  "suma/utils"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - Ribose Inc.
@@ -185,11 +185,14 @@ files:
 - bin/setup
 - exe/suma
 - lib/suma.rb
+- lib/suma/anchor_validator.rb
+- lib/suma/artifact_server.rb
 - lib/suma/build_plan.rb
 - lib/suma/change_classification.rb
 - lib/suma/change_impact.rb
 - lib/suma/change_manifest.rb
 - lib/suma/cli.rb
+- lib/suma/cli/artifact_server_cli.rb
 - lib/suma/cli/build.rb
 - lib/suma/cli/check_svg_quality.rb
 - lib/suma/cli/compare.rb
@@ -199,6 +202,8 @@ files:
 - lib/suma/cli/extract_terms.rb
 - lib/suma/cli/generate_register.rb
 - lib/suma/cli/generate_schemas.rb
+- lib/suma/cli/ingest_translations.rb
+- lib/suma/cli/package_gcr.rb
 - lib/suma/cli/reformat.rb
 - lib/suma/cli/resolve.rb
 - lib/suma/cli/validate.rb
@@ -213,17 +218,20 @@ files:
 - lib/suma/eengine_converter.rb
 - lib/suma/express_reformatter.rb
 - lib/suma/express_schema.rb
+- lib/suma/gcr_package.rb
 - lib/suma/jsdai.rb
 - lib/suma/jsdai/figure.rb
 - lib/suma/jsdai/figure_image.rb
 - lib/suma/jsdai/figure_xml.rb
 - lib/suma/link_validation.rb
+- lib/suma/link_validation_result.rb
 - lib/suma/link_validator.rb
 - lib/suma/link_validator/step.rb
 - lib/suma/manifest_traverser.rb
 - lib/suma/mapping_drift.rb
 - lib/suma/note_processor.rb
 - lib/suma/null_cache.rb
+- lib/suma/output_canonicalizer.rb
 - lib/suma/owl_projection.rb
 - lib/suma/processor.rb
 - lib/suma/provenance.rb
@@ -241,8 +249,10 @@ files:
 - lib/suma/schema_naming.rb
 - lib/suma/schema_profile.rb
 - lib/suma/schema_template.rb
+- lib/suma/schema_template/anchors.rb
 - lib/suma/schema_template/document.rb
 - lib/suma/schema_template/plain.rb
+- lib/suma/site_build.rb
 - lib/suma/site_config.rb
 - lib/suma/staged_collection_builder.rb
 - lib/suma/svg_quality.rb
@@ -256,6 +266,7 @@ files:
 - lib/suma/term_classification.rb
 - lib/suma/term_extractor.rb
 - lib/suma/thor_ext.rb
+- lib/suma/translation_ingester.rb
 - lib/suma/urn.rb
 - lib/suma/urn_resolver.rb
 - lib/suma/utils.rb