suma 0.5.9 → 0.5.11

data/lib/suma/corpus_graph.rb

@@ -13,6 +13,12 @@ module Suma
   # change-impact analysis (S3) and dependency-driven builds (S6). Nodes are
   # schemas and their declared elements; edges are USE/REFERENCE interfaces.
   class CorpusGraph
+    # One USE/REFERENCE import: the target schema, its kind, and the named
+    # items (renames already applied — the visible name is +items+).
+    InterfaceEdge = Struct.new(:kind, :target, :items, keyword_init: true)
+
+    EMPTY_VISIBLE = {}.freeze
+
     attr_reader :schema_ids
 
     def initialize(repositories)
@@ -24,6 +30,7 @@ module Suma
         repo.schemas.each { |schema| add_schema(schema) }
       end
       @schema_ids = @schemas.keys
+      @visibility = build_visibility
       freeze
     end
 
@@ -47,20 +54,18 @@ module Suma
       @schemas.key?(normalize(id))
     end
 
-    # Does +schema_id+ (or a schema it interfaces to) declare +element_id+?
-    # EXPRESS resolves unqualified names against the local schema and
-    # implicitly-used schemas.
+    # Does +schema_id+ (or the schemas it imports) declare +element_id+?
+    # EXPRESS resolves unqualified names against the local schema and the
+    # schema's interface visibility, including USE-FROM cascades.
     #
     # @param schema_id [String]
     # @param element_id [String]
     # @return [Boolean]
     def element_visible?(schema_id, element_id)
       key = normalize(schema_id)
-      return false unless @elements.key?(key)
-
-      return true if @elements[key].key?(normalize(element_id))
+      return false unless @visibility.key?(key)
 
-      visible_from_interfaces(key).include?(normalize(element_id))
+      @visibility[key].key?(normalize(element_id))
     end
 
     # Schemas that +schema_id+ USEs FROM or REFERENCEs.
@@ -68,7 +73,7 @@ module Suma
     # @param schema_id [String]
     # @return [Array<String>] normalized ids of interface targets
     def interface_targets(schema_id)
-      @interfaces[normalize(schema_id)] || []
+      (@interfaces[normalize(schema_id)] || []).map(&:target)
     end
 
     # Transitive interface closure: every schema +schema_id+ depends on,
@@ -110,16 +115,26 @@ module Suma
       key = normalize(schema.id)
       @schemas[key] = true
       @elements[key] = declared_elements(schema)
-      targets = (schema.interfaces || [])
-        .filter_map { |i| i.schema&.id }
-        .map { |id| normalize(id) }
-        .uniq
-      @interfaces[key] = targets
+      @interfaces[key] = Array(schema.interfaces).filter_map do |interface|
+        build_edge(interface)
+      end
+    end
+
+    def build_edge(interface)
+      target = interface.schema&.id
+      return nil unless target
+
+      items = Array(interface.items).filter_map do |item|
+        item.id || (item.ref.id if item.ref.respond_to?(:id))
+      end
+      InterfaceEdge.new(kind: interface.kind.to_s,
+                        target: normalize(target),
+                        items: items)
     end
 
     def declared_elements(schema)
       index = {}
-      schema.children.each do |element|
+      Array(schema.children).each do |element|
         if element.respond_to?(:id) && element.id
           index[normalize(element.id)] =
             true
@@ -128,9 +143,57 @@ module Suma
       index
     end
 
-    def visible_from_interfaces(key)
-      (@interfaces[key] || []).flat_map do |target|
-        @elements[target]&.keys || []
+    # EXPRESS interface visibility (ISO 10303-11 §13.3): a schema sees its
+    # declarations plus its imports. An unrestricted USE re-exports
+    # everything its target makes visible, so visibility cascades through
+    # USE chains; a named import exposes only the named (renamed) items;
+    # a REFERENCE exposes the target's declarations without cascading.
+    #
+    # WG12 corpora ship circular USE interfaces, so this is a monotone
+    # fixed-point iteration — sound under cycles, and each pass only adds.
+    def build_visibility
+      visible = {}
+      @schemas.each_key { |key| visible[key] = (@elements[key] || {}).dup }
+
+      changed = true
+      while changed
+        changed = false
+        @schemas.each_key do |key|
+          changed ||= apply_visibility_pass(key, visible)
+        end
+      end
+      visible
+    end
+
+    def apply_visibility_pass(key, visible)
+      names = visible[key]
+      changed = false
+      (@interfaces[key] || []).each do |edge|
+        next unless @schemas.key?(edge.target)
+
+        changed ||= import_edge_names(names, edge, visible)
+      end
+      changed
+    end
+
+    def import_edge_names(names, edge, visible)
+      changed = false
+      imported_names(edge, visible).each_key do |name|
+        next if names.key?(name)
+
+        names[name] = true
+        changed = true
+      end
+      changed
+    end
+
+    def imported_names(edge, visible)
+      if edge.items.any?
+        edge.items.to_h { |name| [normalize(name), true] }
+      elsif edge.kind == "USE"
+        visible[edge.target] || EMPTY_VISIBLE
+      else
+        @elements[edge.target] || EMPTY_VISIBLE
       end
     end
 

data/lib/suma/gcr_package.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "zip"
+require "glossarist"
+require "fileutils"
+
+module Suma
+  # Packages extracted Glossarist concepts into a GCR (Glossarist Concept
+  # Repository) archive — the sealed distribution format consumed by the
+  # Glossarist Vocabulary Browser (docs/gcr-spec.md in concept-browser).
+  #
+  # Standard variant only: metadata.yaml + register.yaml + concepts/ in the
+  # browser's canonical flat format (termid, term, {lang} blocks). The -full
+  # variant (SKOS/TBX/JSONL compilations) is out of scope.
+  class GcrPackage
+    SCHEMA_VERSION = "1.0.0"
+
+    attr_reader :shortname, :version
+
+    # @param concepts_dir [String] TermExtractor output directory
+    # @param register_file [String] register.yaml (dataset identity)
+    # @param shortname [String] dataset id (datasets.yml `id`)
+    # @param version [String] semver of this dataset release
+    # @param title [String] human-readable dataset title
+    # @param description [String] dataset description
+    # @param created_at [Time, nil] package timestamp; pinned packages pass
+    #   a fixed time so the archive is byte-reproducible
+    def initialize(concepts_dir:, register_file:, shortname:, version:,
+                   title:, description:, created_at: nil)
+      @concepts_dir = File.expand_path(concepts_dir)
+      @register_file = File.expand_path(register_file)
+      @shortname = shortname
+      @version = version
+      @title = title
+      @description = description
+      @created_at = created_at || Time.now
+    end
+
+    # Write the versioned and unversioned .gcr archives.
+    #
+    # @param output_dir [String]
+    # @return [Array<String>] written archive paths
+    def write(output_dir)
+      FileUtils.mkdir_p(output_dir)
+      entries = package_entries
+      [versioned_name, unversioned_name].map do |name|
+        path = File.join(output_dir, name)
+        write_zip(path, entries)
+        Utils.log "Wrote GCR package: #{path}"
+        path
+      end
+    end
+
+    # Canonical concept hashes, in deterministic (termid) order.
+    #
+    # @return [Array<Hash>]
+    def canonical_concepts
+      @canonical_concepts ||= concept_documents
+        .map { |document| canonicalize(document) }
+        .sort_by { |hash| hash["termid"] }
+    end
+
+    # @return [Hash] metadata.yaml content
+    def metadata
+      register = Glossarist::DatasetRegister.from_file(@register_file)
+      {
+        "shortname" => @shortname,
+        "version" => @version,
+        "title" => @title,
+        "description" => @description,
+        "glossarist_version" => Glossarist::VERSION,
+        "created_at" => @created_at.utc.iso8601,
+        "created_by" => "suma CLI",
+        # Top-level summary keys as consumed by the Glossarist Vocabulary
+        # Browser and the Glossarist::GcrMetadata model.
+        "concept_count" => statistics["total_concepts"],
+        "languages" => statistics["languages"],
+        "uri" => register.urn,
+        "uri_aliases" => register.urn_aliases,
+        "statistics" => statistics,
+        "schema_version" => SCHEMA_VERSION,
+      }
+    end
+
+    def statistics
+      languages = canonical_concepts
+        .flat_map { |hash| hash.keys - %w[termid term] }
+        .uniq.sort
+      {
+        "total_concepts" => canonical_concepts.length,
+        "languages" => languages,
+        "concepts_with_definitions" => concepts_with("definition"),
+        "concepts_with_sources" => concepts_with("sources"),
+        "concepts_by_status" => { "valid" => canonical_concepts.length },
+      }
+    end
+
+    def concepts_with(key)
+      canonical_concepts.count do |hash|
+        hash.values.any? { |block| block.is_a?(Hash) && Array(block[key]).any? }
+      end
+    end
+
+    private
+
+    def concept_documents
+      @concept_documents ||= Dir[File.join(@concepts_dir,
+                                           "*.yaml")].map do |path|
+        Glossarist::V3::ConceptDocument.from_yamls(File.read(path))
+      end
+    end
+
+    # Convert a V3 ConceptDocument stream to the browser's canonical flat
+    # concept hash (termid, term, one block per language).
+    def canonicalize(document)
+      concept = document.concept
+      hash = { "termid" => concept.data.id }
+      document.localizations.each do |localized|
+        data = localized.data
+        language = data.language_code
+        hash[language] = language_block(data)
+        hash["term"] ||= preferred_designation(data.terms)
+      end
+      validate_canonical!(hash)
+      hash
+    end
+
+    def language_block(data)
+      block = { "terms" => term_entries(data) }
+      definitions = Array(data.definition)
+      unless definitions.empty?
+        block["definition"] =
+          definition_entries(definitions)
+      end
+      notes = Array(data.notes).map do |note|
+        note.respond_to?(:content) ? note.content : note.to_s
+      end
+      block["notes"] = notes unless notes.empty?
+      sources = Array(data.sources).map { |source| source_hash(source) }
+      block["sources"] = sources unless sources.empty?
+      block["language_code"] = data.language_code
+      block
+    end
+
+    def term_entries(data)
+      Array(data.terms).map do |term|
+        {
+          "type" => term.type,
+          "designation" => term.designation,
+          "normative_status" => term.normative_status,
+        }
+      end
+    end
+
+    def definition_entries(definitions)
+      definitions.map { |definition| { "content" => definition.content } }
+    end
+
+    def source_hash(source)
+      ref = source.origin&.ref
+      origin = {}
+      if ref
+        origin["ref"] =
+          { "source" => ref.source, "version" => ref.version }.compact
+      end
+      { "type" => source.type, "origin" => origin }
+    end
+
+    def preferred_designation(terms)
+      preferred = Array(terms).find do |term|
+        term.normative_status == "preferred"
+      end
+      (preferred || Array(terms).first)&.designation
+    end
+
+    def validate_canonical!(hash)
+      termid = hash["termid"]
+      unless termid.is_a?(String) && !termid.empty?
+        raise Error, "concept without termid"
+      end
+
+      return if language_blocks(hash).any? do |block|
+        Array(block["terms"]).any?
+      end
+
+      raise Error, "concept #{termid} has no language block with a term"
+    end
+
+    def language_blocks(hash)
+      hash.values.grep(Hash)
+    end
+
+    def package_entries
+      entries = {
+        "metadata.yaml" => "#{YAML.dump(metadata)}\n",
+        "register.yaml" => File.read(@register_file),
+      }
+      seen = {}
+      canonical_concepts.each do |hash|
+        termid = hash["termid"]
+        if seen[termid]
+          raise Error, "duplicate termid in package: #{termid}"
+        end
+
+        seen[termid] = true
+        entries["concepts/#{termid}.yaml"] = YAML.dump(hash)
+      end
+      entries
+    end
+
+    def versioned_name
+      "#{@shortname}-#{@version}.gcr"
+    end
+
+    def unversioned_name
+      "#{@shortname}.gcr"
+    end
+
+    def write_zip(path, entries)
+      buffer = Zip::OutputStream.write_buffer do |stream|
+        entries.each do |name, content|
+          stream.put_next_entry(name)
+          stream.write(content)
+        end
+      end
+      File.binwrite(path, buffer.string)
+    end
+  end
+end

data/lib/suma/link_validation_result.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Suma
+  # One unresolved link reported by a validator: where it was found, what
+  # it pointed at, and why it does not resolve.
+  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
+                                    keyword_init: true)
+end

data/lib/suma/link_validator.rb

@@ -4,9 +4,6 @@ require "expressir"
 require "suma/link_validation"
 
 module Suma
-  LinkValidationResult = Struct.new(:file, :line, :link, :reason,
-                                    keyword_init: true)
-
   class LinkValidator
     autoload :Step, "suma/link_validator/step"
 

data/lib/suma/output_canonicalizer.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "digest"
+require "uuidtools"
+
+module Suma
+  # Removes Metanorma's nondeterminism from collection output.
+  #
+  # Metanorma assigns a random UUID to every collection manifest node that
+  # lacks an identifier (Manifest#identifier's default) and stamps each
+  # rendered document with a random 8-letter type marker
+  # (collection/xrefprocess). Both leak into the emitted manifest and the XML
+  # output, so two identical builds produce different bytes.
+  #
+  # Manifest identifiers are replaced IN THE MODEL before serialization:
+  # Psych's quoting of the resulting strings is value-dependent (a few
+  # percent of UUID-looking values get single-quoted), so patching the dumped
+  # YAML afterwards would preserve a random quoting decision. Values that
+  # appear in the source YAML are preserved.
+  class OutputCanonicalizer
+    UUID_PATTERN =
+      /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+    FULL_UUID_PATTERN = /\A#{UUID_PATTERN.source}\z/
+    CANONICAL_TYPE = "SUMASUMA"
+    NAMESPACE = UUIDTools::UUID.parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
+
+    def initialize(output_directory:, sources: [])
+      @output_directory = File.expand_path(output_directory)
+      @sources = sources.map { |path| File.expand_path(path) }
+    end
+
+    # Replace generated manifest identifiers with deterministic UUIDv5s.
+    # Must run before the manifest is serialized and the site key computed.
+    def canonicalize_manifest(manifest)
+      canonicalize_node(manifest, "collection-manifest")
+      self
+    end
+
+    # Canonicalize the rendered site's root XML documents.
+    def canonicalize_documents
+      Dir.glob(File.join(@output_directory, "*.xml")).each do |file|
+        canonicalize_document(file)
+      end
+      self
+    end
+
+    private
+
+    def canonicalize_node(node, seed)
+      return unless node
+
+      if generated_uuid?(node.identifier)
+        node.identifier = uuid_v5("suma-manifest:#{seed}").to_s
+      end
+      Array(node.entry).each_with_index do |child, index|
+        canonicalize_node(child, "#{seed}/#{index}")
+      end
+    end
+
+    def generated_uuid?(value)
+      value.to_s.match?(FULL_UUID_PATTERN) && !preserved.include?(value)
+    end
+
+    def canonicalize_document(file)
+      content = File.read(file)
+      canonical = replace_uuids(content, File.basename(file))
+      canonical = replace_type(canonical)
+      File.write(file, canonical) if canonical != content
+    end
+
+    def replace_uuids(content, seed)
+      index = 0
+      content.gsub(UUID_PATTERN) do |uuid|
+        next uuid if preserved.include?(uuid)
+
+        index += 1
+        uuid_v5("suma-document:#{seed}:#{index}").to_s
+      end
+    end
+
+    def replace_type(content)
+      content.sub(/(<metanorma\b[^>]*\btype=")[A-Z]{8}(")/) do
+        "#{Regexp.last_match(1)}#{CANONICAL_TYPE}#{Regexp.last_match(2)}"
+      end
+    end
+
+    # UUIDs that occur in the site's source YAML. Identifiers the author
+    # wrote must survive; only generated ones are replaced. YAML is read as
+    # bytes: an author-supplied file with non-UTF-8 bytes must not fail the
+    # build, and the pattern is ASCII-only.
+    def preserved
+      @preserved ||= yaml_sources.flat_map do |file|
+        File.binread(file).scan(UUID_PATTERN)
+      end.uniq
+    end
+
+    def yaml_sources
+      @sources.select do |file|
+        file.match?(/\.ya?ml\z/i) && File.file?(file)
+      end
+    end
+
+    def uuid_v5(name)
+      UUIDTools::UUID.sha1_create(NAMESPACE, name)
+    end
+  end
+end

data/lib/suma/owl_projection.rb

@@ -167,7 +167,7 @@ module Suma
     # the Expressir model and must not shadow declarations).
     def index_elements
       @schemas.each_with_object({}) do |(schema_key, schema), map|
-        schema.children.each do |element|
+        Array(schema.children).each do |element|
           next unless class_element?(element)
 
           map[[schema_key, normalize(element.id)]] = element
@@ -231,7 +231,7 @@ module Suma
     end
 
     def project_schema(graph, schema_key, schema)
-      schema.children.each do |element|
+      Array(schema.children).each do |element|
         case element
         when Expressir::Model::Declarations::Entity
           project_entity(graph, schema_key, element)
@@ -482,7 +482,7 @@ module Suma
 
     def validate_materialization(projected)
       projected.flat_map do |schema_key, schema|
-        schema.children.filter_map do |element|
+        Array(schema.children).filter_map do |element|
           next unless class_element?(element)
 
           iri = element_iri(schema_key, element.id)