suma 0.5.9 → 0.5.10

data/lib/suma/gcr_package.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "zip"
+require "glossarist"
+require "fileutils"
+
+module Suma
+  # Packages extracted Glossarist concepts into a GCR (Glossarist Concept
+  # Repository) archive — the sealed distribution format consumed by the
+  # Glossarist Vocabulary Browser (docs/gcr-spec.md in concept-browser).
+  #
+  # Standard variant only: metadata.yaml + register.yaml + concepts/ in the
+  # browser's canonical flat format (termid, term, {lang} blocks). The -full
+  # variant (SKOS/TBX/JSONL compilations) is out of scope.
+  class GcrPackage
+    SCHEMA_VERSION = "1.0.0"
+
+    attr_reader :shortname, :version
+
+    # @param concepts_dir [String] TermExtractor output directory
+    # @param register_file [String] register.yaml (dataset identity)
+    # @param shortname [String] dataset id (datasets.yml `id`)
+    # @param version [String] semver of this dataset release
+    # @param title [String] human-readable dataset title
+    # @param description [String] dataset description
+    # @param created_at [Time, nil] package timestamp; pinned packages pass
+    #   a fixed time so the archive is byte-reproducible
+    def initialize(concepts_dir:, register_file:, shortname:, version:,
+                   title:, description:, created_at: nil)
+      @concepts_dir = File.expand_path(concepts_dir)
+      @register_file = File.expand_path(register_file)
+      @shortname = shortname
+      @version = version
+      @title = title
+      @description = description
+      @created_at = created_at || Time.now
+    end
+
+    # Write the versioned and unversioned .gcr archives.
+    #
+    # @param output_dir [String]
+    # @return [Array<String>] written archive paths
+    def write(output_dir)
+      FileUtils.mkdir_p(output_dir)
+      entries = package_entries
+      [versioned_name, unversioned_name].map do |name|
+        path = File.join(output_dir, name)
+        write_zip(path, entries)
+        Utils.log "Wrote GCR package: #{path}"
+        path
+      end
+    end
+
+    # Canonical concept hashes, in deterministic (termid) order.
+    #
+    # @return [Array<Hash>]
+    def canonical_concepts
+      @canonical_concepts ||= concept_documents
+        .map { |document| canonicalize(document) }
+        .sort_by { |hash| hash["termid"] }
+    end
+
+    # @return [Hash] metadata.yaml content
+    def metadata
+      register = Glossarist::DatasetRegister.from_file(@register_file)
+      {
+        "shortname" => @shortname,
+        "version" => @version,
+        "title" => @title,
+        "description" => @description,
+        "glossarist_version" => Glossarist::VERSION,
+        "created_at" => @created_at.utc.iso8601,
+        "created_by" => "suma CLI",
+        # Top-level summary keys as consumed by the Glossarist Vocabulary
+        # Browser and the Glossarist::GcrMetadata model.
+        "concept_count" => statistics["total_concepts"],
+        "languages" => statistics["languages"],
+        "uri" => register.urn,
+        "uri_aliases" => register.urn_aliases,
+        "statistics" => statistics,
+        "schema_version" => SCHEMA_VERSION,
+      }
+    end
+
+    def statistics
+      languages = canonical_concepts
+        .flat_map { |hash| hash.keys - %w[termid term] }
+        .uniq.sort
+      {
+        "total_concepts" => canonical_concepts.length,
+        "languages" => languages,
+        "concepts_with_definitions" => concepts_with("definition"),
+        "concepts_with_sources" => concepts_with("sources"),
+        "concepts_by_status" => { "valid" => canonical_concepts.length },
+      }
+    end
+
+    def concepts_with(key)
+      canonical_concepts.count do |hash|
+        hash.values.any? { |block| block.is_a?(Hash) && Array(block[key]).any? }
+      end
+    end
+
+    private
+
+    def concept_documents
+      @concept_documents ||= Dir[File.join(@concepts_dir,
+                                           "*.yaml")].map do |path|
+        Glossarist::V3::ConceptDocument.from_yamls(File.read(path))
+      end
+    end
+
+    # Convert a V3 ConceptDocument stream to the browser's canonical flat
+    # concept hash (termid, term, one block per language).
+    def canonicalize(document)
+      concept = document.concept
+      hash = { "termid" => concept.data.id }
+      document.localizations.each do |localized|
+        data = localized.data
+        language = data.language_code
+        hash[language] = language_block(data)
+        hash["term"] ||= preferred_designation(data.terms)
+      end
+      validate_canonical!(hash)
+      hash
+    end
+
+    def language_block(data)
+      block = { "terms" => term_entries(data) }
+      definitions = Array(data.definition)
+      unless definitions.empty?
+        block["definition"] =
+          definition_entries(definitions)
+      end
+      notes = Array(data.notes).map do |note|
+        note.respond_to?(:content) ? note.content : note.to_s
+      end
+      block["notes"] = notes unless notes.empty?
+      sources = Array(data.sources).map { |source| source_hash(source) }
+      block["sources"] = sources unless sources.empty?
+      block["language_code"] = data.language_code
+      block
+    end
+
+    def term_entries(data)
+      Array(data.terms).map do |term|
+        {
+          "type" => term.type,
+          "designation" => term.designation,
+          "normative_status" => term.normative_status,
+        }
+      end
+    end
+
+    def definition_entries(definitions)
+      definitions.map { |definition| { "content" => definition.content } }
+    end
+
+    def source_hash(source)
+      ref = source.origin&.ref
+      origin = {}
+      if ref
+        origin["ref"] =
+          { "source" => ref.source, "version" => ref.version }.compact
+      end
+      { "type" => source.type, "origin" => origin }
+    end
+
+    def preferred_designation(terms)
+      preferred = Array(terms).find do |term|
+        term.normative_status == "preferred"
+      end
+      (preferred || Array(terms).first)&.designation
+    end
+
+    def validate_canonical!(hash)
+      termid = hash["termid"]
+      unless termid.is_a?(String) && !termid.empty?
+        raise Error, "concept without termid"
+      end
+
+      return if language_blocks(hash).any? do |block|
+        Array(block["terms"]).any?
+      end
+
+      raise Error, "concept #{termid} has no language block with a term"
+    end
+
+    def language_blocks(hash)
+      hash.values.grep(Hash)
+    end
+
+    def package_entries
+      entries = {
+        "metadata.yaml" => "#{YAML.dump(metadata)}\n",
+        "register.yaml" => File.read(@register_file),
+      }
+      seen = {}
+      canonical_concepts.each do |hash|
+        termid = hash["termid"]
+        if seen[termid]
+          raise Error, "duplicate termid in package: #{termid}"
+        end
+
+        seen[termid] = true
+        entries["concepts/#{termid}.yaml"] = YAML.dump(hash)
+      end
+      entries
+    end
+
+    def versioned_name
+      "#{@shortname}-#{@version}.gcr"
+    end
+
+    def unversioned_name
+      "#{@shortname}.gcr"
+    end
+
+    def write_zip(path, entries)
+      buffer = Zip::OutputStream.write_buffer do |stream|
+        entries.each do |name, content|
+          stream.put_next_entry(name)
+          stream.write(content)
+        end
+      end
+      File.binwrite(path, buffer.string)
+    end
+  end
+end

data/lib/suma/link_validation_result.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Suma
+  # One unresolved link reported by a validator: where it was found, what
+  # it pointed at, and why it does not resolve.
+  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
+                                    keyword_init: true)
+end

data/lib/suma/link_validator.rb

@@ -4,9 +4,6 @@ require "expressir"
 require "suma/link_validation"
 
 module Suma
-  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
-                                    keyword_init: true)
-
   class LinkValidator
     autoload :Step, "suma/link_validator/step"
 

data/lib/suma/output_canonicalizer.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "digest"
+require "uuidtools"
+
+module Suma
+  # Removes Metanorma's nondeterminism from collection output.
+  #
+  # Metanorma assigns a random UUID to every collection manifest node that
+  # lacks an identifier (Manifest#identifier's default) and stamps each
+  # rendered document with a random 8-letter type marker
+  # (collection/xrefprocess). Both leak into the emitted manifest and the XML
+  # output, so two identical builds produce different bytes.
+  #
+  # Manifest identifiers are replaced IN THE MODEL before serialization:
+  # Psych's quoting of the resulting strings is value-dependent (a few
+  # percent of UUID-looking values get single-quoted), so patching the dumped
+  # YAML afterwards would preserve a random quoting decision. Values that
+  # appear in the source YAML are preserved.
+  class OutputCanonicalizer
+    UUID_PATTERN =
+      /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+    FULL_UUID_PATTERN = /\A#{UUID_PATTERN.source}\z/
+    CANONICAL_TYPE = "SUMASUMA"
+    NAMESPACE = UUIDTools::UUID.parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+
+    def initialize(output_directory:, sources: [])
+      @output_directory = File.expand_path(output_directory)
+      @sources = sources.map { |path| File.expand_path(path) }
+    end
+
+    # Replace generated manifest identifiers with deterministic UUIDv5s.
+    # Must run before the manifest is serialized and the site key computed.
+    def canonicalize_manifest(manifest)
+      canonicalize_node(manifest, "collection-manifest")
+      self
+    end
+
+    # Canonicalize the rendered site's root XML documents.
+    def canonicalize_documents
+      Dir.glob(File.join(@output_directory, "*.xml")).each do |file|
+        canonicalize_document(file)
+      end
+      self
+    end
+
+    private
+
+    def canonicalize_node(node, seed)
+      return unless node
+
+      if generated_uuid?(node.identifier)
+        node.identifier = uuid_v5("suma-manifest:#{seed}").to_s
+      end
+      Array(node.entry).each_with_index do |child, index|
+        canonicalize_node(child, "#{seed}/#{index}")
+      end
+    end
+
+    def generated_uuid?(value)
+      value.to_s.match?(FULL_UUID_PATTERN) && !preserved.include?(value)
+    end
+
+    def canonicalize_document(file)
+      content = File.read(file)
+      canonical = replace_uuids(content, File.basename(file))
+      canonical = replace_type(canonical)
+      File.write(file, canonical) if canonical != content
+    end
+
+    def replace_uuids(content, seed)
+      index = 0
+      content.gsub(UUID_PATTERN) do |uuid|
+        next uuid if preserved.include?(uuid)
+
+        index += 1
+        uuid_v5("suma-document:#{seed}:#{index}").to_s
+      end
+    end
+
+    def replace_type(content)
+      content.sub(/(<metanorma\b[^>]*\btype=")[A-Z]{8}(")/) do
+        "#{Regexp.last_match(1)}#{CANONICAL_TYPE}#{Regexp.last_match(2)}"
+      end
+    end
+
+    # UUIDs that occur in the site's source YAML. Identifiers the author
+    # wrote must survive; only generated ones are replaced. YAML is read as
+    # bytes: an author-supplied file with non-UTF-8 bytes must not fail the
+    # build, and the pattern is ASCII-only.
+    def preserved
+      @preserved ||= yaml_sources.flat_map do |file|
+        File.binread(file).scan(UUID_PATTERN)
+      end.uniq
+    end
+
+    def yaml_sources
+      @sources.select do |file|
+        file.match?(/\.ya?ml\z/i) && File.file?(file)
+      end
+    end
+
+    def uuid_v5(name)
+      UUIDTools::UUID.sha1_create(NAMESPACE, name)
+    end
+  end
+end

data/lib/suma/processor.rb

@@ -14,7 +14,8 @@ module Suma
     def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
                    output_directory: "_site", staged: false,
                    incremental: false, state_file: nil, jobs: 1,
-                   provenance_file: nil, sign_method: nil, sign_key: nil)
+                   provenance_file: nil, sign_method: nil, sign_key: nil,
+                   no_valid: false)
       @metanorma_yaml_path = metanorma_yaml_path
       @schemas_all_path = schemas_all_path
       @compile_flag = compile
@@ -28,6 +29,9 @@ module Suma
       @provenance_file = provenance_file
       @sign_method = sign_method
       @sign_key = sign_key
+      # Skip Metanorma document validation (metanorma `--no-valid`). Opt-in:
+      # for collections whose flavor's validators are still in flight.
+      @no_valid = no_valid
     end
 
     def run
@@ -39,7 +43,6 @@ module Suma
         Utils.log "Compiling schema collection..."
         compile_schema(schemas_all_path, collection_config)
 
-        Utils.log "Compiling complete collection..."
         compile_collection(collection_config, output_directory)
       end
 
@@ -56,6 +59,7 @@ module Suma
         @provenance_file,
         inputs: provenance_inputs,
         outputs: provenance_outputs,
+        exclude: provenance_excludes,
       )
       if @sign_method
         signature = Provenance.sign(
@@ -80,6 +84,31 @@ module Suma
        @output_directory]
     end
 
+    # Build metadata that is not site content: the site freshness state, the
+    # staged-build store, and a provenance file written inside the output tree.
+    def provenance_excludes
+      excludes = [File.join(@output_directory, SiteBuild::STATE_FILE)]
+      if @staged
+        excludes << File.join(@output_directory,
+                              StagedCollectionBuilder::STORE_DIRNAME)
+        excludes << File.join(@output_directory, "_staged_reinflate")
+      end
+      excludes + provenance_artifacts.select { |path| within_output?(path) }
+    end
+
+    def provenance_artifacts
+      return [] unless @provenance_file
+
+      base = File.expand_path(@provenance_file)
+      [base, "#{base}.minisig", "#{base}.bundle"]
+    end
+
+    def within_output?(path)
+      full = File.expand_path(path)
+      output = File.expand_path(@output_directory)
+      full == output || full.start_with?("#{output}#{File::SEPARATOR}")
+    end
+
     def export_schema_config
       site_config = Suma::SiteConfig::Config.from_file(metanorma_yaml_path)
 
@@ -108,6 +137,7 @@ module Suma
           incremental: @incremental,
           state_file: @state_file,
           jobs: @jobs,
+          no_valid: @no_valid,
         },
       )
 
@@ -115,36 +145,80 @@ module Suma
     end
 
     def compile_collection(collection_config, output_directory)
-      metanorma_collection, collection_opts = build_collection(
-        collection_config, output_directory
-      )
+      prepare_collection_manifest(collection_config)
+      write_collection_manifest(collection_config)
+
+      if site_build.current?
+        Utils.log "Collection outputs are up to date; 0 documents rendered."
+        return
+      end
+
+      Utils.log "Rendering collection site..."
+      render_collection(collection_config, output_directory)
+      canonicalizer.canonicalize_documents
+      site_build.record
+    end
+
+    # Generated manifest identifiers are canonicalized in the model, before
+    # serialization: patching the dumped YAML would preserve Psych's
+    # value-dependent (random) quoting decision.
+    def prepare_collection_manifest(collection_config)
+      ManifestTraverser.new(collection_config.manifest)
+        .remove_schemas_only_sources
+      canonicalizer.canonicalize_manifest(collection_config.manifest)
+    end
 
+    def write_collection_manifest(collection_config)
+      collection_config.to_file(COLLECTION_OUTPUT_PATH)
+    end
+
+    def render_collection(collection_config, output_directory)
       if @staged
-        # build_collection has written the emitted manifest; stage each member in
-        # its own process, then reinflate. Bounds peak memory to a single member.
+        # The emitted manifest is the staging input; each member is compiled
+        # in its own process and then reinflated into +output_directory+.
         StagedCollectionBuilder.new(
           collection_config_path: COLLECTION_OUTPUT_PATH,
           output_directory: output_directory,
-          coverpage: collection_opts[:coverpage],
+          coverpage: collection_coverpage(collection_config),
+          no_valid: @no_valid,
         ).build
       else
-        metanorma_collection.render(collection_opts)
+        Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
+          .render(collection_options(collection_config, output_directory))
       end
     end
 
-    def build_collection(collection_config, output_directory)
-      new_collection_config_path = COLLECTION_OUTPUT_PATH
-      ManifestTraverser.new(collection_config.manifest).remove_schemas_only_sources
-      collection_config.to_file(new_collection_config_path)
-
-      collection = Metanorma::Collection.parse(new_collection_config_path)
-
-      collection_opts = {
+    def collection_options(collection_config, output_directory)
+      {
         output_folder: output_directory,
-        compile: { install_fonts: false },
-        coverpage: collection_config.coverpage || "cover.html",
+        compile: compile_flags,
+        coverpage: collection_coverpage(collection_config),
       }
-      [collection, collection_opts]
+    end
+
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
+    def collection_coverpage(collection_config)
+      collection_config.coverpage || "cover.html"
+    end
+
+    def site_build
+      @site_build ||= SiteBuild.new(
+        root: File.dirname(File.expand_path(metanorma_yaml_path)),
+        output_directory: output_directory,
+        inputs: [schemas_all_path],
+      )
+    end
+
+    def canonicalizer
+      @canonicalizer ||= OutputCanonicalizer.new(
+        output_directory: output_directory,
+        sources: site_build.source_files,
+      )
     end
   end
 end

data/lib/suma/provenance.rb

@@ -83,6 +83,9 @@ module Suma
       end
     end
 
+    # Gems whose versions determine rendered document bytes.
+    RENDER_GEM_PREFIXES = %w[metanorma- isodoc].freeze
+
     # @return [Hash] toolchain versions that determine output bytes
     def self.toolchain
       {
@@ -98,30 +101,63 @@ module Suma
       versions
     end
 
+    # Toolchain identity for cached rendered documents: the provenance
+    # toolchain plus every installed Metanorma/Isodoc gem, so a renderer or
+    # flavor update invalidates cached renders even when source is unchanged.
+    def self.render_toolchain
+      @render_toolchain ||= toolchain.merge("render" => render_gems)
+    end
+
+    def self.render_gems
+      gem_specs.select { |spec| render_gem?(spec.name) }
+        .sort_by { |spec| [spec.name, spec.version.to_s] }
+        .map { |spec| "#{spec.name}-#{spec.version}" }
+    end
+
+    def self.render_gem?(name)
+      name == "metanorma" || RENDER_GEM_PREFIXES.any? do |prefix|
+        name.start_with?(prefix)
+      end
+    end
+
+    def self.gem_specs
+      Gem::Specification.select { |spec| spec.name.match?(/\A(?:metanorma|isodoc)/) }
+    end
+
     # @param inputs [Array<String>] files or directories
     # @param outputs [Array<String>] files or directories
+    # @param exclude [Array<String>] files or directories omitted from the
+    #   manifest and from the unrecorded-output check
     # @return [Hash] manifest
-    def self.record(inputs:, outputs:)
+    def self.record(inputs:, outputs:, exclude: [])
+      excluded = exclude.map { |path| File.expand_path(path) }
       {
         "toolchain" => toolchain,
-        "inputs" => digests(inputs),
-        "outputs" => digests(outputs),
+        "inputs" => digests(inputs, exclude: excluded),
+        "outputs" => digests(outputs, exclude: excluded),
+        "output_roots" => output_roots(outputs, exclude: excluded),
+        "excluded" => excluded.map { |path| relative(path) }.sort,
       }
     end
 
     # Write a provenance manifest as canonical JSON.
-    def self.write(path, inputs:, outputs:)
+    def self.write(path, inputs:, outputs:, exclude: [])
       FileUtils.mkdir_p(File.dirname(path))
-      manifest = record(inputs: inputs, outputs: outputs)
+      manifest = record(inputs: inputs, outputs: outputs, exclude: exclude)
       File.write(path, "#{JSON.pretty_generate(manifest)}\n")
       path
     end
 
-    # Independently recompute every digest in the manifest.
+    # Independently recompute every digest in the manifest and flag files
+    # that appeared under a recorded output root but are not in the manifest.
     #
     # @return [Array<String>] problems (empty = verified)
     def self.verify(path)
       manifest = JSON.parse(File.read(path))
+      digest_problems(manifest) + unrecorded_outputs(manifest)
+    end
+
+    def self.digest_problems(manifest)
       SECTIONS.flat_map do |section|
         manifest.fetch(section, {}).filter_map do |relative, digest|
           full = File.expand_path(relative)
@@ -134,18 +170,59 @@ module Suma
       end
     end
 
+    def self.unrecorded_outputs(manifest)
+      recorded = manifest.fetch("outputs", {})
+      excluded = expanded(manifest["excluded"])
+      Array(manifest["output_roots"]).flat_map do |root|
+        output_files(root)
+          .reject { |file| recorded.key?(file) || excluded?(file, excluded) }
+          .map { |file| "outputs: unrecorded #{file}" }
+      end.sort
+    end
+
+    def self.output_files(root)
+      full_root = File.expand_path(root)
+      return [] unless File.directory?(full_root)
+
+      files([full_root]).map { |file| relative(file) }.uniq
+    end
+
+    def self.expanded(paths)
+      Array(paths).map { |path| File.expand_path(path) }
+    end
+
+    def self.excluded?(relative_path, excluded)
+      full = File.expand_path(relative_path)
+      excluded.any? do |path|
+        full == path || full.start_with?("#{path}#{File::SEPARATOR}")
+      end
+    end
+
     # @param paths [Array<String>] files or directories
+    # @param exclude [Array<String>] expanded paths to skip
     # @return [Hash{String=>String}] relative path → sha256
-    def self.digests(paths)
-      files = paths.flat_map do |path|
-        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
-      end
-      files.select { |file| File.file?(file) }
+    def self.digests(paths, exclude: [])
+      files(paths)
+        .reject { |file| excluded?(file, exclude) }
         .map { |file| relative(file) }
         .uniq.sort
         .to_h { |file| [file, Digest::SHA256.file(file).hexdigest] }
     end
 
+    def self.files(paths)
+      paths.flat_map do |path|
+        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
+      end.select { |file| File.file?(file) }
+    end
+
+    # Relative paths of the directory entries among +outputs+.
+    def self.output_roots(outputs, exclude: [])
+      outputs.select { |path| File.directory?(path) }
+        .map { |path| relative(path) }
+        .reject { |path| excluded?(path, exclude) }
+        .uniq.sort
+    end
+
     def self.relative(path)
       Pathname.new(File.expand_path(path))
         .relative_path_from(Pathname.getwd)
@@ -154,6 +231,9 @@ module Suma
       path.to_s
     end
 
-    private_class_method :optional_versions, :sign_with, :executable?
+    private_class_method :optional_versions, :sign_with, :executable?,
+                         :render_gems, :render_gem?, :gem_specs,
+                         :digest_problems, :unrecorded_outputs, :output_files,
+                         :expanded, :excluded?, :files
   end
 end

data/lib/suma/schema_collection.rb

@@ -125,6 +125,7 @@ module Suma
         schema: express,
         output_path: @output_path_docs.join(express.id),
         template: template_class.new(express.id),
+        no_valid: @options[:no_valid],
       )
 
       @docs[express.id] = compiler