suma 0.5.8 → 0.5.10

data/lib/suma/corpus_graph.rb

@@ -13,6 +13,12 @@ module Suma
   # change-impact analysis (S3) and dependency-driven builds (S6). Nodes are
   # schemas and their declared elements; edges are USE/REFERENCE interfaces.
   class CorpusGraph
+    # One USE/REFERENCE import: the target schema, its kind, and the named
+    # items (renames already applied — the visible name is +items+).
+    InterfaceEdge = Struct.new(:kind, :target, :items, keyword_init: true)
+
+    EMPTY_VISIBLE = {}.freeze
+
     attr_reader :schema_ids
 
     def initialize(repositories)
@@ -24,6 +30,7 @@ module Suma
         repo.schemas.each { |schema| add_schema(schema) }
       end
       @schema_ids = @schemas.keys
+      @visibility = build_visibility
       freeze
     end
 
@@ -47,20 +54,18 @@ module Suma
       @schemas.key?(normalize(id))
     end
 
-    # Does +schema_id+ (or a schema it interfaces to) declare +element_id+?
-    # EXPRESS resolves unqualified names against the local schema and
-    # implicitly-used schemas.
+    # Does +schema_id+ (or the schemas it imports) declare +element_id+?
+    # EXPRESS resolves unqualified names against the local schema and the
+    # schema's interface visibility, including USE-FROM cascades.
     #
     # @param schema_id [String]
     # @param element_id [String]
     # @return [Boolean]
     def element_visible?(schema_id, element_id)
       key = normalize(schema_id)
-      return false unless @elements.key?(key)
-
-      return true if @elements[key].key?(normalize(element_id))
+      return false unless @visibility.key?(key)
 
-      visible_from_interfaces(key).include?(normalize(element_id))
+      @visibility[key].key?(normalize(element_id))
     end
 
     # Schemas that +schema_id+ USEs FROM or REFERENCEs.
@@ -68,7 +73,7 @@ module Suma
     # @param schema_id [String]
     # @return [Array<String>] normalized ids of interface targets
     def interface_targets(schema_id)
-      @interfaces[normalize(schema_id)] || []
+      (@interfaces[normalize(schema_id)] || []).map(&:target)
     end
 
     # Transitive interface closure: every schema +schema_id+ depends on,
@@ -110,11 +115,21 @@ module Suma
       key = normalize(schema.id)
       @schemas[key] = true
       @elements[key] = declared_elements(schema)
-      targets = (schema.interfaces || [])
-        .filter_map { |i| i.schema&.id }
-        .map { |id| normalize(id) }
-        .uniq
-      @interfaces[key] = targets
+      @interfaces[key] = Array(schema.interfaces).filter_map do |interface|
+        build_edge(interface)
+      end
+    end
+
+    def build_edge(interface)
+      target = interface.schema&.id
+      return nil unless target
+
+      items = Array(interface.items).filter_map do |item|
+        item.id || (item.ref.id if item.ref.respond_to?(:id))
+      end
+      InterfaceEdge.new(kind: interface.kind.to_s,
+                        target: normalize(target),
+                        items: items)
     end
 
     def declared_elements(schema)
@@ -128,9 +143,57 @@ module Suma
       index
     end
 
-    def visible_from_interfaces(key)
-      (@interfaces[key] || []).flat_map do |target|
-        @elements[target]&.keys || []
+    # EXPRESS interface visibility (ISO 10303-11 §13.3): a schema sees its
+    # declarations plus its imports. An unrestricted USE re-exports
+    # everything its target makes visible, so visibility cascades through
+    # USE chains; a named import exposes only the named (renamed) items;
+    # a REFERENCE exposes the target's declarations without cascading.
+    #
+    # WG12 corpora ship circular USE interfaces, so this is a monotone
+    # fixed-point iteration — sound under cycles, and each pass only adds.
+    def build_visibility
+      visible = {}
+      @schemas.each_key { |key| visible[key] = (@elements[key] || {}).dup }
+
+      changed = true
+      while changed
+        changed = false
+        @schemas.each_key do |key|
+          changed ||= apply_visibility_pass(key, visible)
+        end
+      end
+      visible
+    end
+
+    def apply_visibility_pass(key, visible)
+      names = visible[key]
+      changed = false
+      (@interfaces[key] || []).each do |edge|
+        next unless @schemas.key?(edge.target)
+
+        changed ||= import_edge_names(names, edge, visible)
+      end
+      changed
+    end
+
+    def import_edge_names(names, edge, visible)
+      changed = false
+      imported_names(edge, visible).each_key do |name|
+        next if names.key?(name)
+
+        names[name] = true
+        changed = true
+      end
+      changed
+    end
+
+    def imported_names(edge, visible)
+      if edge.items.any?
+        edge.items.to_h { |name| [normalize(name), true] }
+      elsif edge.kind == "USE"
+        visible[edge.target] || EMPTY_VISIBLE
+      else
+        @elements[edge.target] || EMPTY_VISIBLE
       end
     end
 

data/lib/suma/gcr_package.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "zip"
+require "glossarist"
+require "fileutils"
+
+module Suma
+  # Packages extracted Glossarist concepts into a GCR (Glossarist Concept
+  # Repository) archive — the sealed distribution format consumed by the
+  # Glossarist Vocabulary Browser (docs/gcr-spec.md in concept-browser).
+  #
+  # Standard variant only: metadata.yaml + register.yaml + concepts/ in the
+  # browser's canonical flat format (termid, term, {lang} blocks). The -full
+  # variant (SKOS/TBX/JSONL compilations) is out of scope.
+  class GcrPackage
+    SCHEMA_VERSION = "1.0.0"
+
+    attr_reader :shortname, :version
+
+    # @param concepts_dir [String] TermExtractor output directory
+    # @param register_file [String] register.yaml (dataset identity)
+    # @param shortname [String] dataset id (datasets.yml `id`)
+    # @param version [String] semver of this dataset release
+    # @param title [String] human-readable dataset title
+    # @param description [String] dataset description
+    # @param created_at [Time, nil] package timestamp; pinned packages pass
+    #   a fixed time so the archive is byte-reproducible
+    def initialize(concepts_dir:, register_file:, shortname:, version:,
+                   title:, description:, created_at: nil)
+      @concepts_dir = File.expand_path(concepts_dir)
+      @register_file = File.expand_path(register_file)
+      @shortname = shortname
+      @version = version
+      @title = title
+      @description = description
+      @created_at = created_at || Time.now
+    end
+
+    # Write the versioned and unversioned .gcr archives.
+    #
+    # @param output_dir [String]
+    # @return [Array<String>] written archive paths
+    def write(output_dir)
+      FileUtils.mkdir_p(output_dir)
+      entries = package_entries
+      [versioned_name, unversioned_name].map do |name|
+        path = File.join(output_dir, name)
+        write_zip(path, entries)
+        Utils.log "Wrote GCR package: #{path}"
+        path
+      end
+    end
+
+    # Canonical concept hashes, in deterministic (termid) order.
+    #
+    # @return [Array<Hash>]
+    def canonical_concepts
+      @canonical_concepts ||= concept_documents
+        .map { |document| canonicalize(document) }
+        .sort_by { |hash| hash["termid"] }
+    end
+
+    # @return [Hash] metadata.yaml content
+    def metadata
+      register = Glossarist::DatasetRegister.from_file(@register_file)
+      {
+        "shortname" => @shortname,
+        "version" => @version,
+        "title" => @title,
+        "description" => @description,
+        "glossarist_version" => Glossarist::VERSION,
+        "created_at" => @created_at.utc.iso8601,
+        "created_by" => "suma CLI",
+        # Top-level summary keys as consumed by the Glossarist Vocabulary
+        # Browser and the Glossarist::GcrMetadata model.
+        "concept_count" => statistics["total_concepts"],
+        "languages" => statistics["languages"],
+        "uri" => register.urn,
+        "uri_aliases" => register.urn_aliases,
+        "statistics" => statistics,
+        "schema_version" => SCHEMA_VERSION,
+      }
+    end
+
+    def statistics
+      languages = canonical_concepts
+        .flat_map { |hash| hash.keys - %w[termid term] }
+        .uniq.sort
+      {
+        "total_concepts" => canonical_concepts.length,
+        "languages" => languages,
+        "concepts_with_definitions" => concepts_with("definition"),
+        "concepts_with_sources" => concepts_with("sources"),
+        "concepts_by_status" => { "valid" => canonical_concepts.length },
+      }
+    end
+
+    def concepts_with(key)
+      canonical_concepts.count do |hash|
+        hash.values.any? { |block| block.is_a?(Hash) && Array(block[key]).any? }
+      end
+    end
+
+    private
+
+    def concept_documents
+      @concept_documents ||= Dir[File.join(@concepts_dir,
+                                           "*.yaml")].map do |path|
+        Glossarist::V3::ConceptDocument.from_yamls(File.read(path))
+      end
+    end
+
+    # Convert a V3 ConceptDocument stream to the browser's canonical flat
+    # concept hash (termid, term, one block per language).
+    def canonicalize(document)
+      concept = document.concept
+      hash = { "termid" => concept.data.id }
+      document.localizations.each do |localized|
+        data = localized.data
+        language = data.language_code
+        hash[language] = language_block(data)
+        hash["term"] ||= preferred_designation(data.terms)
+      end
+      validate_canonical!(hash)
+      hash
+    end
+
+    def language_block(data)
+      block = { "terms" => term_entries(data) }
+      definitions = Array(data.definition)
+      unless definitions.empty?
+        block["definition"] =
+          definition_entries(definitions)
+      end
+      notes = Array(data.notes).map do |note|
+        note.respond_to?(:content) ? note.content : note.to_s
+      end
+      block["notes"] = notes unless notes.empty?
+      sources = Array(data.sources).map { |source| source_hash(source) }
+      block["sources"] = sources unless sources.empty?
+      block["language_code"] = data.language_code
+      block
+    end
+
+    def term_entries(data)
+      Array(data.terms).map do |term|
+        {
+          "type" => term.type,
+          "designation" => term.designation,
+          "normative_status" => term.normative_status,
+        }
+      end
+    end
+
+    def definition_entries(definitions)
+      definitions.map { |definition| { "content" => definition.content } }
+    end
+
+    def source_hash(source)
+      ref = source.origin&.ref
+      origin = {}
+      if ref
+        origin["ref"] =
+          { "source" => ref.source, "version" => ref.version }.compact
+      end
+      { "type" => source.type, "origin" => origin }
+    end
+
+    def preferred_designation(terms)
+      preferred = Array(terms).find do |term|
+        term.normative_status == "preferred"
+      end
+      (preferred || Array(terms).first)&.designation
+    end
+
+    def validate_canonical!(hash)
+      termid = hash["termid"]
+      unless termid.is_a?(String) && !termid.empty?
+        raise Error, "concept without termid"
+      end
+
+      return if language_blocks(hash).any? do |block|
+        Array(block["terms"]).any?
+      end
+
+      raise Error, "concept #{termid} has no language block with a term"
+    end
+
+    def language_blocks(hash)
+      hash.values.grep(Hash)
+    end
+
+    def package_entries
+      entries = {
+        "metadata.yaml" => "#{YAML.dump(metadata)}\n",
+        "register.yaml" => File.read(@register_file),
+      }
+      seen = {}
+      canonical_concepts.each do |hash|
+        termid = hash["termid"]
+        if seen[termid]
+          raise Error, "duplicate termid in package: #{termid}"
+        end
+
+        seen[termid] = true
+        entries["concepts/#{termid}.yaml"] = YAML.dump(hash)
+      end
+      entries
+    end
+
+    def versioned_name
+      "#{@shortname}-#{@version}.gcr"
+    end
+
+    def unversioned_name
+      "#{@shortname}.gcr"
+    end
+
+    def write_zip(path, entries)
+      buffer = Zip::OutputStream.write_buffer do |stream|
+        entries.each do |name, content|
+          stream.put_next_entry(name)
+          stream.write(content)
+        end
+      end
+      File.binwrite(path, buffer.string)
+    end
+  end
+end

data/lib/suma/link_validation_result.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Suma
+  # One unresolved link reported by a validator: where it was found, what
+  # it pointed at, and why it does not resolve.
+  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
+                                    keyword_init: true)
+end

data/lib/suma/link_validator.rb

@@ -4,9 +4,6 @@ require "expressir"
 require "suma/link_validation"
 
 module Suma
-  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
-                                    keyword_init: true)
-
   class LinkValidator
     autoload :Step, "suma/link_validator/step"
 

data/lib/suma/output_canonicalizer.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "digest"
+require "uuidtools"
+
+module Suma
+  # Removes Metanorma's nondeterminism from collection output.
+  #
+  # Metanorma assigns a random UUID to every collection manifest node that
+  # lacks an identifier (Manifest#identifier's default) and stamps each
+  # rendered document with a random 8-letter type marker
+  # (collection/xrefprocess). Both leak into the emitted manifest and the XML
+  # output, so two identical builds produce different bytes.
+  #
+  # Manifest identifiers are replaced IN THE MODEL before serialization:
+  # Psych's quoting of the resulting strings is value-dependent (a few
+  # percent of UUID-looking values get single-quoted), so patching the dumped
+  # YAML afterwards would preserve a random quoting decision. Values that
+  # appear in the source YAML are preserved.
+  class OutputCanonicalizer
+    UUID_PATTERN =
+      /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+    FULL_UUID_PATTERN = /\A#{UUID_PATTERN.source}\z/
+    CANONICAL_TYPE = "SUMASUMA"
+    NAMESPACE = UUIDTools::UUID.parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+
+    def initialize(output_directory:, sources: [])
+      @output_directory = File.expand_path(output_directory)
+      @sources = sources.map { |path| File.expand_path(path) }
+    end
+
+    # Replace generated manifest identifiers with deterministic UUIDv5s.
+    # Must run before the manifest is serialized and the site key computed.
+    def canonicalize_manifest(manifest)
+      canonicalize_node(manifest, "collection-manifest")
+      self
+    end
+
+    # Canonicalize the rendered site's root XML documents.
+    def canonicalize_documents
+      Dir.glob(File.join(@output_directory, "*.xml")).each do |file|
+        canonicalize_document(file)
+      end
+      self
+    end
+
+    private
+
+    def canonicalize_node(node, seed)
+      return unless node
+
+      if generated_uuid?(node.identifier)
+        node.identifier = uuid_v5("suma-manifest:#{seed}").to_s
+      end
+      Array(node.entry).each_with_index do |child, index|
+        canonicalize_node(child, "#{seed}/#{index}")
+      end
+    end
+
+    def generated_uuid?(value)
+      value.to_s.match?(FULL_UUID_PATTERN) && !preserved.include?(value)
+    end
+
+    def canonicalize_document(file)
+      content = File.read(file)
+      canonical = replace_uuids(content, File.basename(file))
+      canonical = replace_type(canonical)
+      File.write(file, canonical) if canonical != content
+    end
+
+    def replace_uuids(content, seed)
+      index = 0
+      content.gsub(UUID_PATTERN) do |uuid|
+        next uuid if preserved.include?(uuid)
+
+        index += 1
+        uuid_v5("suma-document:#{seed}:#{index}").to_s
+      end
+    end
+
+    def replace_type(content)
+      content.sub(/(<metanorma\b[^>]*\btype=")[A-Z]{8}(")/) do
+        "#{Regexp.last_match(1)}#{CANONICAL_TYPE}#{Regexp.last_match(2)}"
+      end
+    end
+
+    # UUIDs that occur in the site's source YAML. Identifiers the author
+    # wrote must survive; only generated ones are replaced. YAML is read as
+    # bytes: an author-supplied file with non-UTF-8 bytes must not fail the
+    # build, and the pattern is ASCII-only.
+    def preserved
+      @preserved ||= yaml_sources.flat_map do |file|
+        File.binread(file).scan(UUID_PATTERN)
+      end.uniq
+    end
+
+    def yaml_sources
+      @sources.select do |file|
+        file.match?(/\.ya?ml\z/i) && File.file?(file)
+      end
+    end
+
+    def uuid_v5(name)
+      UUIDTools::UUID.sha1_create(NAMESPACE, name)
+    end
+  end
+end

data/lib/suma/processor.rb

@@ -14,7 +14,8 @@ module Suma
     def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
                    output_directory: "_site", staged: false,
                    incremental: false, state_file: nil, jobs: 1,
-                   provenance_file: nil, sign_method: nil, sign_key: nil)
+                   provenance_file: nil, sign_method: nil, sign_key: nil,
+                   no_valid: false)
       @metanorma_yaml_path = metanorma_yaml_path
       @schemas_all_path = schemas_all_path
       @compile_flag = compile
@@ -28,6 +29,9 @@ module Suma
       @provenance_file = provenance_file
       @sign_method = sign_method
       @sign_key = sign_key
+      # Skip Metanorma document validation (metanorma `--no-valid`). Opt-in:
+      # for collections whose flavor's validators are still in flight.
+      @no_valid = no_valid
     end
 
     def run
@@ -39,7 +43,6 @@ module Suma
         Utils.log "Compiling schema collection..."
         compile_schema(schemas_all_path, collection_config)
 
-        Utils.log "Compiling complete collection..."
         compile_collection(collection_config, output_directory)
       end
 
@@ -56,6 +59,7 @@ module Suma
         @provenance_file,
         inputs: provenance_inputs,
         outputs: provenance_outputs,
+        exclude: provenance_excludes,
       )
       if @sign_method
         signature = Provenance.sign(
@@ -80,6 +84,31 @@ module Suma
        @output_directory]
     end
 
+    # Build metadata that is not site content: the site freshness state, the
+    # staged-build store, and a provenance file written inside the output tree.
+    def provenance_excludes
+      excludes = [File.join(@output_directory, SiteBuild::STATE_FILE)]
+      if @staged
+        excludes << File.join(@output_directory,
+                              StagedCollectionBuilder::STORE_DIRNAME)
+        excludes << File.join(@output_directory, "_staged_reinflate")
+      end
+      excludes + provenance_artifacts.select { |path| within_output?(path) }
+    end
+
+    def provenance_artifacts
+      return [] unless @provenance_file
+
+      base = File.expand_path(@provenance_file)
+      [base, "#{base}.minisig", "#{base}.bundle"]
+    end
+
+    def within_output?(path)
+      full = File.expand_path(path)
+      output = File.expand_path(@output_directory)
+      full == output || full.start_with?("#{output}#{File::SEPARATOR}")
+    end
+
     def export_schema_config
       site_config = Suma::SiteConfig::Config.from_file(metanorma_yaml_path)
 
@@ -108,6 +137,7 @@ module Suma
           incremental: @incremental,
           state_file: @state_file,
           jobs: @jobs,
+          no_valid: @no_valid,
         },
       )
 
@@ -115,36 +145,80 @@ module Suma
     end
 
     def compile_collection(collection_config, output_directory)
-      metanorma_collection, collection_opts = build_collection(
-        collection_config, output_directory
-      )
+      prepare_collection_manifest(collection_config)
+      write_collection_manifest(collection_config)
+
+      if site_build.current?
+        Utils.log "Collection outputs are up to date; 0 documents rendered."
+        return
+      end
+
+      Utils.log "Rendering collection site..."
+      render_collection(collection_config, output_directory)
+      canonicalizer.canonicalize_documents
+      site_build.record
+    end
+
+    # Generated manifest identifiers are canonicalized in the model, before
+    # serialization: patching the dumped YAML would preserve Psych's
+    # value-dependent (random) quoting decision.
+    def prepare_collection_manifest(collection_config)
+      ManifestTraverser.new(collection_config.manifest)
+        .remove_schemas_only_sources
+      canonicalizer.canonicalize_manifest(collection_config.manifest)
+    end
 
+    def write_collection_manifest(collection_config)
+      collection_config.to_file(COLLECTION_OUTPUT_PATH)
+    end
+
+    def render_collection(collection_config, output_directory)
       if @staged
-        # build_collection has written the emitted manifest; stage each member in
-        # its own process, then reinflate. Bounds peak memory to a single member.
+        # The emitted manifest is the staging input; each member is compiled
+        # in its own process and then reinflated into +output_directory+.
         StagedCollectionBuilder.new(
           collection_config_path: COLLECTION_OUTPUT_PATH,
           output_directory: output_directory,
-          coverpage: collection_opts[:coverpage],
+          coverpage: collection_coverpage(collection_config),
+          no_valid: @no_valid,
         ).build
       else
-        metanorma_collection.render(collection_opts)
+        Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
+          .render(collection_options(collection_config, output_directory))
       end
     end
 
-    def build_collection(collection_config, output_directory)
-      new_collection_config_path = COLLECTION_OUTPUT_PATH
-      ManifestTraverser.new(collection_config.manifest).remove_schemas_only_sources
-      collection_config.to_file(new_collection_config_path)
-
-      collection = Metanorma::Collection.parse(new_collection_config_path)
-
-      collection_opts = {
+    def collection_options(collection_config, output_directory)
+      {
         output_folder: output_directory,
-        compile: { install_fonts: false },
-        coverpage: collection_config.coverpage || "cover.html",
+        compile: compile_flags,
+        coverpage: collection_coverpage(collection_config),
       }
-      [collection, collection_opts]
+    end
+
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
+    def collection_coverpage(collection_config)
+      collection_config.coverpage || "cover.html"
+    end
+
+    def site_build
+      @site_build ||= SiteBuild.new(
+        root: File.dirname(File.expand_path(metanorma_yaml_path)),
+        output_directory: output_directory,
+        inputs: [schemas_all_path],
+      )
+    end
+
+    def canonicalizer
+      @canonicalizer ||= OutputCanonicalizer.new(
+        output_directory: output_directory,
+        sources: site_build.source_files,
+      )
     end
   end
 end