suma 0.5.8 → 0.5.10

data/lib/suma/site_build.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "fileutils"
+require "pathname"
+
+module Suma
+  # Freshness decision for the rendered collection site.
+  #
+  # The collection render is expensive: Metanorma compiles every member
+  # document. Its output is a pure function of the emitted collection manifest,
+  # the source tree it references, and the rendering toolchain, so this class
+  # keys those inputs, remembers the key and the produced files in the output
+  # directory, and lets the processor skip the render when nothing changed.
+  class SiteBuild
+    STATE_FILE = ".suma-site-build.json"
+
+    # Emitted manifests that the build itself regenerates deterministically
+    # from the sources above; whether they exist yet must not change the key.
+    GENERATED_FILES = %w[collection-output.yaml schemas.yml].freeze
+
+    def initialize(root:, output_directory:, inputs: [])
+      @root = File.expand_path(root)
+      @output_directory = File.expand_path(output_directory)
+      @extra_inputs = inputs.map { |path| File.expand_path(path) }
+    end
+
+    # @return [Array<String>] source files determining the rendered site
+    def source_files
+      @source_files ||= begin
+        files = @extra_inputs.select { |file| File.file?(file) }
+        (files + tree_files).uniq.sort
+      end
+    end
+
+    # @return [String] digest of every source file and the render toolchain
+    def key
+      @key ||= begin
+        digest = Digest::SHA256.new
+        source_files.each do |file|
+          digest << relative(file) << "\0"
+          digest << Digest::SHA256.file(file).hexdigest << "\0"
+        end
+        digest << JSON.generate(Provenance.render_toolchain)
+        digest.hexdigest
+      end
+    end
+
+    # True when the previous render used the same inputs and every file it
+    # produced is still present.
+    def current?
+      state = read_state
+      return false unless state && state["key"] == key
+
+      outputs = Array(state["outputs"])
+      !outputs.empty? &&
+        outputs.all? { |file| File.file?(File.join(@output_directory, file)) }
+    end
+
+    # Persist the key and the produced files for the next build.
+    def record
+      write_state("key" => key, "outputs" => output_files)
+      self
+    end
+
+    def state_path
+      File.join(@output_directory, STATE_FILE)
+    end
+
+    private
+
+    def tree_files
+      excluded = excluded_paths
+      Dir.glob(File.join(@root, "**", "*")).select do |file|
+        File.file?(file) &&
+          excluded.none? { |path| under?(file, path) } &&
+          !generated?(relative(file).tr("\\", "/"))
+      end
+    end
+
+    # Files the collection render writes next to its sources. They are
+    # outputs of the previous build, never inputs, so including them would
+    # change the key on every run.
+    def generated?(relative)
+      base = File.basename(relative)
+      GENERATED_FILES.include?(relative) ||
+        relative.start_with?("relaton/") ||
+        base.start_with?("tmp_document") ||
+        base.end_with?("_index.html", ".err.html", ".log.txt") ||
+        document_artifact?(relative, base)
+    end
+
+    def document_artifact?(relative, base)
+      return false unless relative.start_with?("documents/")
+
+      base == "cover.html" || base == "document.xml" ||
+        (base.start_with?("document.") && base.include?(".xml"))
+    end
+
+    # Generated trees are derived from the sources above and never feed the
+    # site render as inputs of their own. Bundler's install directory is a
+    # dependency artifact: CI conventionally places it at vendor/bundle
+    # inside the project, where it must never count as site source.
+    def excluded_paths
+      fixed = %w[schema_docs plain_schemas .git .bundle vendor/bundle]
+        .map { |dir| File.join(@root, dir) }
+      env = [ENV.fetch("SUMA_SCHEMA_CACHE_DIR", nil),
+             ENV.fetch("BUNDLE_PATH", nil)]
+        .compact
+        .map { |dir| File.expand_path(dir, @root) }
+      [@output_directory] + fixed + env
+    end
+
+    def under?(file, path)
+      file == path || file.start_with?("#{path}#{File::SEPARATOR}")
+    end
+
+    def read_state
+      return unless File.file?(state_path)
+
+      JSON.parse(File.read(state_path))
+    rescue JSON::ParserError
+      nil
+    end
+
+    def write_state(state)
+      FileUtils.mkdir_p(@output_directory)
+      temp = "#{state_path}.#{Process.pid}.tmp"
+      File.write(temp, "#{JSON.pretty_generate(state)}\n")
+      File.rename(temp, state_path)
+    ensure
+      File.delete(temp) if temp && File.exist?(temp)
+    end
+
+    def output_files
+      Dir.glob(File.join(@output_directory, "**", "*"))
+        .select do |file|
+          File.file?(file) && File.basename(file) != STATE_FILE
+        end
+        .map { |file| relative_to(file, @output_directory) }
+        .sort
+    end
+
+    def relative(path)
+      relative_to(path, @root)
+    end
+
+    def relative_to(path, base)
+      Pathname.new(path).relative_path_from(Pathname.new(base)).to_s
+    end
+  end
+end

data/lib/suma/staged_collection_builder.rb

@@ -37,14 +37,16 @@ module Suma
     # +collection_config_path+ is the emitted collection manifest (the same
     # +collection-output.yaml+ the normal build renders). Member filerefs are
     # relative to its directory, so all per-member manifests are written there.
+    # (a data-declaring initializer)
     def initialize(collection_config_path:, output_directory:, coverpage: nil,
-                   store_dir: nil, formats: %i[xml html])
+                   store_dir: nil, formats: %i[xml html], no_valid: false) # rubocop:disable Metrics/ParameterLists
       @config_path = collection_config_path
       @config_dir = File.dirname(File.expand_path(collection_config_path))
       @output_directory = output_directory
       @coverpage = coverpage
       @store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
       @formats = formats
+      @no_valid = no_valid
     end
 
     def build
@@ -97,7 +99,7 @@ module Suma
         manifest_path: one_path,
         opts: { format: [:xml], output_folder: out,
                 preserve_unresolved: true, artifact_store_dir: @store_dir,
-                compile: { install_fonts: false } },
+                compile: compile_flags },
       )
     ensure
       FileUtils.rm_f(one_path)
@@ -118,10 +120,16 @@ module Suma
         manifest_path: reinf_path,
         opts: { format: @formats, output_folder: @output_directory,
                 reinflate: true, coverpage: @coverpage,
-                compile: { install_fonts: false } }.compact,
+                compile: compile_flags }.compact,
       )
     end
 
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
     # Copy each member's stored stub into +dir+ and return docref entries pointing
     # at the copies, keyed by real docid (as the store keys them).
     def stored_docrefs(members, dir)

data/lib/suma/translation_ingester.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "glossarist"
+require "yaml"
+
+module Suma
+  # Overlays translated content onto extracted Glossarist concepts (T5).
+  #
+  # A translations file is a YAML sequence of localized-concept documents,
+  # each carrying an identity key plus the content to apply:
+  #
+  #   - concept_id: example_schema.bolt   # managed id (data.identifier)
+  #                                       # or the managed concept's uuid
+  #   - id: <localized uuid>              # alternative identity key
+  #     data:
+  #       language_code: fra
+  #       terms: [...]
+  #       definition: [...]
+  #
+  # Every field the entry provides replaces the localization's field; every
+  # field it omits falls back, explicitly, to the localization's current
+  # content (for a new language: the source-language localization's). All
+  # (de)serialization goes through the Glossarist/lutaml-model framework;
+  # entries are merged as serialized hashes and reified with of_yaml.
+  class TranslationIngester
+    def initialize(concepts_dir:, translations:, output_dir: nil)
+      @concepts_dir = concepts_dir
+      @translations = translations
+      @output_dir = output_dir || concepts_dir
+    end
+
+    # @return [Hash{String=>Array<String>}] concept file → applied entry
+    #   identities
+    def ingest
+      entries = load_entries
+      applied = {}
+      each_concept_document do |file, doc|
+        matched = entries_for(doc, entries)
+        next if matched.empty?
+
+        applied[file] = matched.map { |entry| apply_entry(doc, entry) }
+        write_document(file, doc)
+      end
+      assert_no_unmatched!(entries, applied)
+      applied
+    end
+
+    private
+
+    def load_entries
+      content = YAML.safe_load_file(@translations, permitted_classes: [],
+                                                   aliases: true)
+      entries = content.is_a?(Hash) ? content["translations"] : content
+      unless entries.is_a?(Array)
+        raise Error,
+              "translations file must be a YAML sequence of localized " \
+              "concepts: #{@translations}"
+      end
+
+      entries
+    end
+
+    def each_concept_document
+      Dir.glob(File.join(@concepts_dir, "*.yaml")).each do |file|
+        doc = Glossarist::V3::ConceptDocument.from_yamls(File.read(file))
+        yield file, doc
+      end
+    end
+
+    def entries_for(doc, entries)
+      entries.select { |entry| entry_matches?(doc, entry) }
+    end
+
+    def entry_matches?(doc, entry)
+      if entry["concept_id"]
+        [doc.concept.data.id, doc.concept.uuid].include?(entry["concept_id"])
+      else
+        localized_uuids(doc).include?(entry["id"])
+      end
+    end
+
+    def localized_uuids(doc)
+      doc.localizations.map(&:id) +
+        doc.concept.data.localized_concepts.values
+    end
+
+    def apply_entry(doc, entry)
+      incoming = Glossarist::LocalizedConcept
+        .of_yaml(entry.except("concept_id"))
+      localized = merge_localization(doc, entry, incoming)
+      replace_localization(doc, localized)
+      entry["concept_id"] || entry["id"]
+    end
+
+    def replace_localization(doc, localized)
+      language = localized.data.language_code
+      doc.localizations.reject! { |l10n| l10n.data.language_code == language }
+      doc.localizations << localized
+      doc.concept.data.localized_concepts[language] = localized.uuid
+    end
+
+    # Entry fields win; absent fields keep the localization's current
+    # content (or the source-language content for a new language). The
+    # merge happens on the raw entry data — a parsed model would
+    # materialize empty collections and wipe the fields it omits.
+    #
+    # Glossarist parses the serialized "id" key into `#id` (not `#uuid`),
+    # so the persisted id is preserved and stamped back onto the merged
+    # localization explicitly; a new language gets the same v5 id
+    # TermExtractor's linked localizations use.
+    def merge_localization(doc, entry, incoming)
+      current = current_localization(doc, entry, incoming)
+      base = current || doc.localizations.first
+      return incoming unless base
+
+      identity = entry["id"] || current&.id ||
+        new_localization_uuid(doc, incoming.data.language_code)
+      localized = Glossarist::LocalizedConcept
+        .of_yaml(merged_hash(base, entry, identity))
+      localized.uuid = identity
+      localized
+    end
+
+    def merged_hash(base, entry, identity)
+      merged = base.to_yaml_hash
+      merged["data"] = (merged["data"] || {})
+        .merge(entry["data"] || {})
+      merged["id"] = identity
+      merged
+    end
+
+    def current_localization(doc, entry, incoming)
+      if entry["id"]
+        doc.localizations.find { |l10n| l10n.id == entry["id"] }
+      else
+        doc.localizations.find do |l10n|
+          l10n.data.language_code == incoming.data.language_code
+        end
+      end
+    end
+
+    # Same convention as TermExtractor's linked localizations.
+    def new_localization_uuid(doc, language)
+      Glossarist::Utilities::UUID.uuid_v5(
+        Glossarist::Utilities::UUID::OID_NAMESPACE,
+        "#{doc.concept.data.id}-#{language}",
+      )
+    end
+
+    def write_document(file, doc)
+      output = File.join(@output_dir, File.basename(file))
+      FileUtils.mkdir_p(@output_dir) if output != file
+      File.write(output, doc.to_yamls, encoding: "utf-8")
+    end
+
+    def assert_no_unmatched!(entries, applied)
+      matched = applied.values.flatten
+      unmatched = entries.reject do |entry|
+        matched.include?(entry["concept_id"] || entry["id"])
+      end
+      return if unmatched.empty?
+
+      raise Error,
+            "unmatched translation entries: " \
+            "#{unmatched.map { |e| e['concept_id'] || e['id'] }.join(', ')}"
+    end
+  end
+end

data/lib/suma/urn_resolver.rb

@@ -53,9 +53,12 @@ module Suma
       )
     end
 
-    # Write JSON and HTML for every artifact.
+    # Write JSON and HTML for every artifact, plus an index of the set.
     def write(output_directory)
       artifacts.each_key { |key| write_urn(key, output_directory) }
+      FileUtils.mkdir_p(output_directory)
+      File.write(File.join(output_directory, "index.html"),
+                 index_document, encoding: "utf-8")
       Utils.log "Wrote #{artifacts.length} URN artifacts to #{output_directory}"
     end
 
@@ -97,11 +100,38 @@ module Suma
           payload = element_payload(schema, element)
           artifacts[compose(:tech, "#{schema.id}.#{element.id}")] = payload if
             payload
+          add_attribute_artifacts(artifacts, schema, element)
         end
       end
       artifacts
     end
 
+    # Attribute property IRIs appear in the OWL projection as
+    # <schema>.<entity>.<attribute>; make them dereferenceable too.
+    def add_attribute_artifacts(artifacts, schema, element)
+      return unless element.is_a?(Expressir::Model::Declarations::Entity)
+
+      element.attributes.each do |attribute|
+        key = compose(:tech,
+                      "#{schema.id}.#{element.id}.#{attribute.id}")
+        artifacts[key] = attribute_artifact(schema, element, attribute)
+      end
+    end
+
+    def attribute_artifact(schema, entity, attribute)
+      {
+        "urn" => compose(:tech, "#{schema.id}.#{entity.id}.#{attribute.id}"),
+        "kind" => "attribute",
+        "schema" => schema.id,
+        "entity" => entity.id,
+        "id" => attribute.id,
+        "attribute_kind" => attribute.kind,
+        "optional" => attribute.optional == true,
+        "type" => attribute.type&.source,
+        "source" => relative_source(schema),
+      }
+    end
+
     def schemas
       @schemas ||= begin
         manifest = Expressir::SchemaManifest.from_file(@manifest_file)
@@ -310,6 +340,26 @@ module Suma
       HTML
     end
 
+    # A deterministic index of every resolved URN, linked to its document.
+    def index_document
+      items = artifacts.keys.sort.map do |urn|
+        href = artifact_path(urn, "html")
+        %(    <li><a href="#{escape_html(href)}">#{escape_html(urn)}</a></li>)
+      end.join("\n")
+      <<~HTML
+        <!doctype html>
+        <html lang="en">
+        <head><meta charset="utf-8"><title>Resolved URNs</title></head>
+        <body>
+        <h1>Resolved URNs</h1>
+        <ul>
+        #{items}
+        </ul>
+        </body>
+        </html>
+      HTML
+    end
+
     def escape_html(value)
       value.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
         .gsub('"', "&quot;")

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.5.8"
+  VERSION = "0.5.10"
 end

data/lib/suma.rb

@@ -7,6 +7,8 @@ require "fileutils"
 module Suma
   autoload :VERSION, "suma/version"
 
+  autoload :AnchorValidator,        "suma/anchor_validator"
+  autoload :ArtifactServer,         "suma/artifact_server"
   autoload :SchemaProfile,          "suma/schema_profile"
   autoload :BuildPlan,              "suma/build_plan"
   autoload :Processor,              "suma/processor"
@@ -20,14 +22,17 @@ module Suma
   autoload :CorpusGraph,            "suma/corpus_graph"
   autoload :CollectionManifest,     "suma/collection_manifest"
   autoload :EengineConverter,       "suma/eengine_converter"
+  autoload :GcrPackage,             "suma/gcr_package"
   autoload :ExpressReformatter,     "suma/express_reformatter"
   autoload :ExpressSchema,          "suma/express_schema"
   autoload :LinkValidation,         "suma/link_validation"
+  autoload :LinkValidationResult,   "suma/link_validation_result"
   autoload :LinkValidator,          "suma/link_validator"
   autoload :ManifestTraverser,      "suma/manifest_traverser"
   autoload :MappingDrift,           "suma/mapping_drift"
   autoload :NoteProcessor,          "suma/note_processor"
   autoload :NullCache,              "suma/null_cache"
+  autoload :OutputCanonicalizer,    "suma/output_canonicalizer"
   autoload :OwlProjection,          "suma/owl_projection"
   autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
   autoload :SchemaCache,            "suma/schema_cache"
@@ -42,11 +47,13 @@ module Suma
   autoload :SchemaNaming,           "suma/schema_naming"
   autoload :SchemaTemplate,         "suma/schema_template"
   autoload :SiteConfig,             "suma/site_config"
+  autoload :SiteBuild,              "suma/site_build"
   autoload :StagedCollectionBuilder, "suma/staged_collection_builder"
   autoload :SvgQuality,             "suma/svg_quality"
   autoload :TermClassification,     "suma/term_classification"
   autoload :TermExtractor,          "suma/term_extractor"
   autoload :ThorExt,                "suma/thor_ext"
+  autoload :TranslationIngester,    "suma/translation_ingester"
   autoload :Urn,                    "suma/urn"
   autoload :UrnResolver,            "suma/urn_resolver"
   autoload :Utils,                  "suma/utils"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.10
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-10-20 00:00:00.000000000 Z
+date: 2026-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -185,11 +185,14 @@ files:
 - bin/setup
 - exe/suma
 - lib/suma.rb
+- lib/suma/anchor_validator.rb
+- lib/suma/artifact_server.rb
 - lib/suma/build_plan.rb
 - lib/suma/change_classification.rb
 - lib/suma/change_impact.rb
 - lib/suma/change_manifest.rb
 - lib/suma/cli.rb
+- lib/suma/cli/artifact_server_cli.rb
 - lib/suma/cli/build.rb
 - lib/suma/cli/check_svg_quality.rb
 - lib/suma/cli/compare.rb
@@ -199,6 +202,8 @@ files:
 - lib/suma/cli/extract_terms.rb
 - lib/suma/cli/generate_register.rb
 - lib/suma/cli/generate_schemas.rb
+- lib/suma/cli/ingest_translations.rb
+- lib/suma/cli/package_gcr.rb
 - lib/suma/cli/reformat.rb
 - lib/suma/cli/resolve.rb
 - lib/suma/cli/validate.rb
@@ -213,17 +218,20 @@ files:
 - lib/suma/eengine_converter.rb
 - lib/suma/express_reformatter.rb
 - lib/suma/express_schema.rb
+- lib/suma/gcr_package.rb
 - lib/suma/jsdai.rb
 - lib/suma/jsdai/figure.rb
 - lib/suma/jsdai/figure_image.rb
 - lib/suma/jsdai/figure_xml.rb
 - lib/suma/link_validation.rb
+- lib/suma/link_validation_result.rb
 - lib/suma/link_validator.rb
 - lib/suma/link_validator/step.rb
 - lib/suma/manifest_traverser.rb
 - lib/suma/mapping_drift.rb
 - lib/suma/note_processor.rb
 - lib/suma/null_cache.rb
+- lib/suma/output_canonicalizer.rb
 - lib/suma/owl_projection.rb
 - lib/suma/processor.rb
 - lib/suma/provenance.rb
@@ -241,8 +249,10 @@ files:
 - lib/suma/schema_naming.rb
 - lib/suma/schema_profile.rb
 - lib/suma/schema_template.rb
+- lib/suma/schema_template/anchors.rb
 - lib/suma/schema_template/document.rb
 - lib/suma/schema_template/plain.rb
+- lib/suma/site_build.rb
 - lib/suma/site_config.rb
 - lib/suma/staged_collection_builder.rb
 - lib/suma/svg_quality.rb
@@ -256,6 +266,7 @@ files:
 - lib/suma/term_classification.rb
 - lib/suma/term_extractor.rb
 - lib/suma/thor_ext.rb
+- lib/suma/translation_ingester.rb
 - lib/suma/urn.rb
 - lib/suma/urn_resolver.rb
 - lib/suma/utils.rb