suma 0.5.7 → 0.5.9

data/bin/benchmark_corpus

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Nightly scale benchmark: generates a synthetic EXPRESS corpus, then times
+# the corpus-scale operations (parse/graph, closure, OWL projection) and
+# asserts wall-time and peak-RSS budgets. Budgets are environment-tunable so
+# CI machines can be calibrated without code changes.
+
+require "suma"
+require "tmpdir"
+require "fileutils"
+require "json"
+
+module CorpusBenchmark
+  SCHEMA_COUNT = Integer(ENV.fetch("SUMA_BENCH_SCHEMAS", "1000"))
+  MAX_SECONDS = Float(ENV.fetch("SUMA_BENCH_MAX_SECONDS", "180"))
+  MAX_RSS_MB = Integer(ENV.fetch("SUMA_BENCH_MAX_RSS_MB", "2048"))
+  URN = "urn:example:benchmark:corpus:ed-1:en"
+
+  module_function
+
+  # Half resources, half modules; each module USEs one resource so the graph
+  # has interface edges (dependents are exercised by the closure walk).
+  def generate(directory)
+    resources = SCHEMA_COUNT / 2
+    FileUtils.mkdir_p(File.join(directory, "resources"))
+    FileUtils.mkdir_p(File.join(directory, "modules"))
+    manifest = { "schemas" => {} }
+
+    resources.times do |index|
+      id = format("res_%04d", index)
+      write_schema(directory, "resources", id, resource_source(id, index))
+      manifest["schemas"][id] = { "path" => "resources/#{id}.exp" }
+    end
+
+    (SCHEMA_COUNT - resources).times do |index|
+      id = format("mod_%04d", index)
+      resource = format("res_%04d", index)
+      write_schema(directory, "modules", id,
+                   module_source(id, index, resource))
+      manifest["schemas"][id] = { "path" => "modules/#{id}.exp" }
+    end
+
+    path = File.join(directory, "schemas.yaml")
+    File.write(path, YAML.dump(manifest))
+    path
+  end
+
+  def resource_source(id, index)
+    <<~EXPRESS
+      SCHEMA #{id};
+      ENTITY thing_#{format('%04d', index)};
+        name : STRING;
+        weight : REAL;
+      END_ENTITY;
+      END_SCHEMA;
+    EXPRESS
+  end
+
+  def module_source(id, index, resource)
+    <<~EXPRESS
+      SCHEMA #{id};
+      USE FROM #{resource};
+      ENTITY wrapper_#{format('%04d', index)};
+        item : thing_#{format('%04d', index)};
+      END_ENTITY;
+      END_SCHEMA;
+    EXPRESS
+  end
+
+  def write_schema(directory, kind, id, source)
+    File.write(File.join(directory, kind, "#{id}.exp"), source)
+  end
+
+  def measure
+    started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    value = yield
+    [value, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started]
+  end
+
+  def peak_rss_mb
+    if File.exist?("/proc/self/status")
+      File.read("/proc/self/status")[/VmHWM:\s+(\d+)/, 1].to_i / 1024.0
+    else
+      `ps -o rss= -p #{Process.pid}`.to_i / 1024.0
+    end.round
+  end
+
+  def run
+    Dir.mktmpdir("suma-benchmark") do |directory|
+      started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      manifest = generate(directory)
+
+      graph, graph_seconds = measure { Suma::CorpusGraph.from_manifest(manifest) }
+      violations, closure_seconds = measure do
+        Suma::ClosureValidator.new(graph).validate_manifest(manifest)
+      end
+      projection, owl_seconds = measure do
+        Suma::OwlProjection.new(manifest, urn: URN)
+      end
+      turtle, turtle_seconds = measure { projection.to_turtle }
+      total = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+      rss = peak_rss_mb
+
+      report = {
+        schemas: SCHEMA_COUNT,
+        graph_seconds: graph_seconds.round(2),
+        closure_seconds: closure_seconds.round(2),
+        owl_seconds: owl_seconds.round(2),
+        turtle_seconds: turtle_seconds.round(2),
+        total_seconds: total.round(2),
+        peak_rss_mb: rss,
+        closure_violations: violations.length,
+        turtle_bytes: turtle.bytesize,
+        budgets: { max_seconds: MAX_SECONDS, max_rss_mb: MAX_RSS_MB },
+      }
+      puts JSON.pretty_generate(report)
+
+      failures = []
+      unless violations.empty?
+        failures << "#{violations.length} closure violations"
+      end
+      if total > MAX_SECONDS
+        failures << "wall time #{total.round(2)}s > #{MAX_SECONDS}s"
+      end
+      if rss > MAX_RSS_MB
+        failures << "peak RSS #{rss}MB > #{MAX_RSS_MB}MB"
+      end
+      unless failures.empty?
+        warn "benchmark FAILED: #{failures.join('; ')}"
+        exit 1
+      end
+      puts "benchmark passed"
+    end
+  end
+end
+
+CorpusBenchmark.run if $PROGRAM_NAME == __FILE__

data/bin/release_provenance

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Emit provenance for a release build: the built gem's sha256 plus the
+# dependency lockfile and toolchain versions. Run after `gem build`.
+
+require "suma"
+
+ROOT = File.expand_path("..", __dir__)
+gem_file = Dir[File.join(ROOT, "*.gem")].max
+abort "release_provenance: no built gem found in #{ROOT}" unless gem_file
+
+manifest_path = File.join(Dir.pwd, "provenance.json")
+Suma::Provenance.write(
+  manifest_path,
+  inputs: [
+    File.join(ROOT, "Gemfile.lock"),
+    File.join(ROOT, "suma.gemspec"),
+  ],
+  outputs: [gem_file],
+)
+
+problems = Suma::Provenance.verify(manifest_path)
+unless problems.empty?
+  warn "release_provenance: verification failed: #{problems.join('; ')}"
+  exit 1
+end
+
+puts "Wrote #{manifest_path} (#{File.basename(gem_file)})"
+
+sign_method = ENV.fetch("SUMA_PROVENANCE_SIGN", nil)
+sign_method = nil if sign_method.to_s.empty?
+if sign_method
+  sign_key = ENV.fetch("SUMA_PROVENANCE_SIGN_KEY", nil)
+  sign_key = nil if sign_key.to_s.empty?
+  signature = Suma::Provenance.sign(
+    manifest_path, method: sign_method, key: sign_key
+  )
+  puts "Signed provenance (#{sign_method}): #{signature}"
+end

data/lib/suma/build_plan.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "fileutils"
+
+module Suma
+  # Decides which schemas an incremental build must re-render.
+  #
+  # The state is a map of schema id → sha256 of its source. A schema is
+  # changed when its digest differs from the previous build; rendering is
+  # required for every changed schema plus all transitive dependents, because
+  # a dependent's cross-references and rendered anchors may shift. The
+  # dependents come from the corpus graph (S1), the same USE/REFERENCE edges
+  # that drive change impact (S3).
+  class BuildPlan
+    # @param entries [Array<#id,#path>] schema manifest entries
+    # @param graph [CorpusGraph]
+    # @param previous_state [Hash{String=>String}] digest per schema id
+    def initialize(entries:, graph:, previous_state: {})
+      @entries = entries
+      @graph = graph
+      @previous_state = previous_state || {}
+      @ids_by_key = entries.to_h { |entry| [entry.id.downcase, entry.id] }
+    end
+
+    # @return [Array<String>] declared ids whose source differs since the
+    #   previous build (all ids on a first build)
+    def changed
+      current_state.reject do |id, digest|
+        @previous_state[id] == digest
+      end.keys
+    end
+
+    # @return [Array<String>] schemas to re-render: every changed schema plus
+    #   its transitive dependents; empty when nothing changed
+    def rebuild
+      changed_ids = changed
+      return [] if changed_ids.empty?
+
+      expanded = changed_ids.flat_map do |id|
+        [id] + @graph.dependents_of(id).map { |key| @ids_by_key[key] || key }
+      end
+      expanded.uniq
+    end
+
+    # @return [Hash{String=>String}] digest per schema id
+    def current_state
+      @current_state ||= @entries.to_h do |entry|
+        [entry.id, Digest::SHA256.file(entry.path.to_s).hexdigest]
+      end
+    end
+
+    # Persist the current state for the next incremental build.
+    def self.write_state(path, state)
+      FileUtils.mkdir_p(File.dirname(path))
+      temp = "#{path}.#{Process.pid}.tmp"
+      File.write(temp, JSON.pretty_generate(state))
+      File.rename(temp, path)
+    ensure
+      File.delete(temp) if temp && File.exist?(temp)
+    end
+
+    # @return [Hash{String=>String}] the previous state, empty when absent
+    def self.read_state(path)
+      return {} unless path && File.exist?(path)
+
+      JSON.parse(File.read(path))
+    end
+  end
+end

data/lib/suma/change_classification.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Suma
+  # The typed change taxonomy (S3): every schema change is classified as
+  # additive, compatible, breaking, or editorial.
+  #
+  # The mapping is data — a frozen Hash keyed by the eengine action — so a
+  # finer-grained rule is a one-line addition. Modifications are classified
+  # as breaking conservatively: eengine reports structural deltas (type
+  # changes, attribute additions on entities, interface changes) without
+  # enough detail to prove backwards compatibility, so additive/editorial
+  # claims are only made where the action itself guarantees them.
+  class ChangeClassification
+    ADDITIVE = "additive"
+    COMPATIBLE = "compatible"
+    BREAKING = "breaking"
+    EDITORIAL = "editorial"
+
+    BY_ACTION = {
+      "added" => ADDITIVE,
+      "removed" => BREAKING,
+      "modified" => BREAKING,
+      "mapping" => EDITORIAL,
+    }.freeze
+
+    ALL = [ADDITIVE, COMPATIBLE, BREAKING, EDITORIAL].freeze
+
+    # @param action [String, Symbol] added / removed / modified / mapping
+    # @return [String] one of ALL
+    def self.classify(action:)
+      BY_ACTION.fetch(action.to_s) do
+        raise Error, "[suma] unknown change action #{action.inspect} " \
+                     "(known: #{BY_ACTION.keys.sort.join(', ')})"
+      end
+    end
+  end
+end

data/lib/suma/change_impact.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "expressir"
+require "pathname"
+
+module Suma
+  # Change-impact analysis (S3): given a changed schema, which schemas and
+  # AP publications are affected?
+  #
+  # Walks the reverse interface closure of the CorpusGraph, so the answer
+  # is transitive: every schema that USEs FROM the changed schema directly
+  # or through other schemas is reported, and every AP publication whose
+  # dependency closure contains the changed schema is flagged.
+  class ChangeImpact
+    attr_reader :graph, :schema_manifest_file
+
+    # @param graph [CorpusGraph]
+    # @param schema_manifest_file [String] corpus manifest, for AP detection
+    def initialize(graph, schema_manifest_file)
+      @graph = graph
+      @schema_manifest_file = schema_manifest_file
+    end
+
+    # @param schema_id [String] the changed schema
+    # @return [Array<String>] transitively affected schema ids
+    def affected_schemas(schema_id)
+      graph.dependents_of(schema_id)
+    end
+
+    # @param schema_id [String] the changed schema
+    # @return [Array<String>] AP publication leads whose closure includes it
+    def affected_publications(schema_id)
+      target = schema_id.to_s.downcase
+      application_protocol_ids.select do |lead|
+        graph.dependency_closure(lead).include?(target)
+      end.sort
+    end
+
+    # @param schema_id [String]
+    # @return [Hash]
+    def report(schema_id)
+      {
+        "schema" => schema_id,
+        "affected_schemas" => affected_schemas(schema_id),
+        "affected_publications" => affected_publications(schema_id),
+      }
+    end
+
+    private
+
+    def application_protocol_ids
+      manifest = Expressir::SchemaManifest.from_file(
+        File.expand_path(schema_manifest_file),
+      )
+      manifest.schemas.select do |entry|
+        SchemaCategory.for_schema(id: entry.id, path: entry.path) ==
+          SchemaCategory::APPLICATION_PROTOCOLS
+      end.map { |entry| entry.id.downcase }
+    end
+  end
+end

data/lib/suma/change_manifest.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "lutaml/model"
+require "expressir/changes"
+
+module Suma
+  # One typed change entry: which element changed, how, and whether the
+  # change is additive/compatible/breaking/editorial.
+  class ChangeEntry < Lutaml::Model::Serializable
+    attribute :element, :string
+    attribute :element_type, :string
+    attribute :action, :string
+    attribute :classification, :string
+    attribute :description, :string, collection: true
+
+    yaml do
+      map "element", to: :element
+      map "element_type", to: :element_type
+      map "action", to: :action
+      map "classification", to: :classification
+      map "description", to: :description
+    end
+  end
+
+  # The typed change manifest for one schema edition pair (S3): the
+  # machine-readable companion to eengine's .changes.yaml.
+  #
+  # Built from the Expressir::Changes::SchemaChange model (the same model
+  # the .changes.yaml embodies), adding the classification dimension and a
+  # rendered changelog.
+  class ChangeManifest < Lutaml::Model::Serializable
+    attribute :schema, :string
+    attribute :version, :integer
+    attribute :entries, ChangeEntry, collection: true
+
+    yaml do
+      map "schema", to: :schema
+      map "version", to: :version
+      map "entries", to: :entries
+    end
+
+    class << self
+      # Build a manifest from a SchemaChange model.
+      #
+      # @param change [Expressir::Changes::SchemaChange]
+      # @param version [Integer, String, nil] version to select
+      #   (default: the latest)
+      # @return [ChangeManifest]
+      def from_schema_change(change, version: nil)
+        selected = select_version(change.versions, version)
+        return new(schema: change.schema) if selected.nil?
+
+        new(
+          schema: change.schema,
+          version: selected.version,
+          entries: entries_for(selected),
+        )
+      end
+
+      # Build a manifest from a .changes.yaml file.
+      #
+      # @param path [String]
+      # @param version [Integer, String, nil]
+      # @return [ChangeManifest]
+      def from_file(path, version: nil)
+        from_schema_change(
+          Expressir::Changes::SchemaChange.from_file(path),
+          version: version,
+        )
+      end
+
+      private
+
+      def select_version(versions, version)
+        return versions.last if version.nil?
+
+        versions.find { |candidate| candidate.version == version.to_i }
+      end
+
+      def entries_for(version_change)
+        entries = []
+        entries.concat(item_entries(version_change.additions, "added"))
+        entries.concat(item_entries(version_change.modifications, "modified"))
+        entries.concat(item_entries(version_change.deletions, "removed"))
+        entries.concat(mapping_entries(version_change.mappings))
+        entries.concat(mapping_entries(version_change.changes))
+        entries
+      end
+
+      def item_entries(items, action)
+        Array(items).map do |item|
+          ChangeEntry.new(
+            element: item.name,
+            element_type: item.type,
+            action: action,
+            classification: ChangeClassification.classify(action: action),
+            description: Array(item.description),
+          )
+        end
+      end
+
+      def mapping_entries(mappings)
+        Array(mappings).map do |mapping|
+          ChangeEntry.new(
+            element: mapping.name,
+            element_type: "MAPPING",
+            action: "mapping",
+            classification: ChangeClassification::EDITORIAL,
+            description: [mapping.description].compact,
+          )
+        end
+      end
+    end
+
+    # Counts per classification.
+    #
+    # @return [Hash{String => Integer}]
+    def summary
+      entries.group_by(&:classification)
+        .transform_values(&:size)
+        .sort.to_h
+    end
+
+    # @return [Boolean]
+    def breaking?
+      entries.any? do |entry|
+        entry.classification == ChangeClassification::BREAKING
+      end
+    end
+
+    # Render a markdown changelog section.
+    #
+    # @return [String]
+    def to_changelog
+      lines = ["## Schema `#{schema}` — version #{version}", ""]
+      entries.sort_by { |entry| [entry.classification, entry.element.to_s] }
+        .each do |entry|
+        lines << "- **#{entry.classification}** — #{entry.element_type} " \
+                 "`#{entry.element}` #{entry.action}"
+        Array(entry.description).each do |description|
+          lines << "  - #{description}"
+        end
+      end
+      lines.join("\n")
+    end
+  end
+end

data/lib/suma/cli/build.rb

@@ -16,6 +16,23 @@ module Suma
                       desc: "Memory-bounded staged build: compile each member in " \
                             "its own process (sequential) and reinflate. For large " \
                             "collections that OOM a single-process build (suma#94)."
+      option :incremental, type: :boolean, default: false,
+                           desc: "Re-render only changed schemas and their " \
+                                 "transitive dependents (state file defaults " \
+                                 "to plain_schemas/.suma-build-state.json)"
+      option :state_file, type: :string,
+                          desc: "Path to the incremental build state file"
+      option :jobs, type: :numeric, default: 1,
+                    desc: "Concurrent schema export workers (document " \
+                          "compilation remains sequential)"
+      option :provenance, type: :string,
+                          desc: "Write a provenance manifest (sha256 of inputs " \
+                                "and outputs + toolchain versions) to this path"
+      option :sign, type: :string, banner: "METHOD",
+                    desc: "Sign the provenance manifest: minisign or sigstore"
+      option :sign_key, type: :string,
+                        desc: "Signing key (minisign requires it; sigstore " \
+                              "signs keylessly when omitted)"
 
       def build(metanorma_site_manifest)
         unless File.exist?(metanorma_site_manifest)
@@ -39,6 +56,12 @@ module Suma
           compile: options[:compile],
           output_directory: "_site",
           staged: options[:staged],
+          incremental: options[:incremental],
+          state_file: options[:state_file],
+          jobs: options[:jobs],
+          provenance_file: options[:provenance],
+          sign_method: options[:sign],
+          sign_key: options[:sign_key],
         ).run
       end
 

data/lib/suma/cli/compare.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "yaml"
+
 require "thor"
 
 module Suma
@@ -39,6 +41,16 @@ module Suma
                                  desc: "Override auto-detected reference repo root"
       option :verbose, type: :boolean, default: false,
                        desc: "Enable verbose output"
+      option :typed_output, type: :string, banner: "FILE",
+                            desc: "Write the typed ChangeManifest " \
+                                  "(classification + descriptions) as YAML"
+      option :changelog, type: :boolean, default: false,
+                         desc: "Print a markdown changelog for the changes"
+      option :impact, type: :boolean, default: false,
+                      desc: "Print affected schemas and AP publications " \
+                            "(requires --manifest)"
+      option :manifest, type: :string, banner: "SCHEMAS_YAML",
+                        desc: "Corpus schema manifest, for --impact"
 
       def compare(trial_schema, reference_schema)
         comparer = SchemaComparer.new(trial_schema, reference_schema, options)
@@ -47,12 +59,47 @@ module Suma
 
         if result.nil?
           say "No changes detected between schemas", :yellow
-        else
-          say "Change YAML file: #{result}", :green
+          return
         end
+
+        say "Change YAML file: #{result}", :green
+        report_typed_changes(result, trial_schema)
       rescue Suma::Error => e
         raise Thor::Error, e.message
       end
+
+      private
+
+      def report_typed_changes(change_yaml, trial_schema)
+        manifest = ChangeManifest.from_file(change_yaml,
+                                            version: options[:version])
+        write_typed_output(manifest)
+        say manifest.to_changelog if options[:changelog]
+        print_impact(manifest, trial_schema) if options[:impact]
+      end
+
+      def write_typed_output(manifest)
+        path = options[:typed_output]
+        return unless path
+
+        File.write(path, manifest.to_yaml)
+        say "Typed change manifest: #{path}", :green
+      end
+
+      def print_impact(manifest, trial_schema)
+        unless options[:manifest]
+          raise Thor::Error, "--impact requires --manifest SCHEMAS_YAML"
+        end
+
+        graph = CorpusGraph.from_manifest(options[:manifest])
+        impact = ChangeImpact.new(graph, options[:manifest])
+          .report(changed_schema_id(manifest, trial_schema))
+        say impact.to_yaml
+      end
+
+      def changed_schema_id(manifest, trial_schema)
+        manifest.schema || File.basename(trial_schema, ".exp")
+      end
     end
   end
 end