suma 0.5.7 → 0.5.9

data/lib/suma/provenance.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "fileutils"
+require "pathname"
+require "expressir"
+
+module Suma
+  # Build provenance: sha256 of every input and output artifact plus the
+  # toolchain versions that produced them.
+  #
+  # Paths are recorded relative to the working directory so a manifest can be
+  # moved with its build root. {.verify} recomputes the digests independently.
+  class Provenance
+    SECTIONS = %w[inputs outputs].freeze
+
+    # Sign a provenance manifest with an external tool.
+    #
+    # Optional by design: callers invoke this only when signing is requested.
+    # minisign uses a secret key (`key:`), sigstore/cosign signs with a key or
+    # keylessly when no key is given (ambient OIDC, e.g. GitHub Actions).
+    #
+    # @param path [String] provenance manifest
+    # @param method [String] minisign or sigstore
+    # @param key [String, nil] secret key path (minisign requires it)
+    # @return [String] signature artifact path
+    def self.sign(path, method:, key: nil)
+      binary, argv, signature = signing_command(
+        method: method, path: path, key: key,
+      )
+      sign_with(binary, argv)
+      signature
+    end
+
+    # The external command and signature path for a signing method. Pure
+    # apart from filesystem-free validation; exposed for cross-platform specs.
+    #
+    # @return [Array(String, Array<String>, String)] binary, argv, signature
+    def self.signing_command(method:, path:, key: nil)
+      case method.to_s
+      when "minisign"
+        unless key
+          raise ArgumentError, "minisign signing requires a secret key"
+        end
+
+        signature = "#{path}.minisig"
+        ["minisign", ["minisign", "-S", "-W", "-s", key, "-m", path,
+                      "-x", signature], signature]
+      when "sigstore"
+        signature = "#{path}.bundle"
+        argv = ["cosign", "sign-blob", "--yes",
+                "--bundle", signature]
+        argv += ["--key", key] if key
+        argv << path
+        ["cosign", argv, signature]
+      else
+        raise ArgumentError, "unsupported signing method: #{method}"
+      end
+    end
+
+    # @return [Boolean] whether the signing tool is on PATH
+    def self.signing_available?(method)
+      case method.to_s
+      when "minisign" then executable?("minisign")
+      when "sigstore" then executable?("cosign")
+      else false
+      end
+    end
+
+    def self.sign_with(binary, argv)
+      unless executable?(binary)
+        raise Error, "#{binary} is not available on PATH"
+      end
+      return if system(*argv)
+
+      raise Error, "#{binary} signing failed: #{argv.join(' ')}"
+    end
+
+    def self.executable?(binary)
+      ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).any? do |directory|
+        File.executable?(File.join(directory, binary))
+      end
+    end
+
+    # @return [Hash] toolchain versions that determine output bytes
+    def self.toolchain
+      {
+        "ruby" => RUBY_VERSION,
+        "expressir" => Expressir::VERSION,
+        "suma" => Suma::VERSION,
+      }.merge(optional_versions)
+    end
+
+    def self.optional_versions
+      versions = {}
+      versions["metanorma"] = Metanorma::VERSION if defined?(Metanorma::VERSION)
+      versions
+    end
+
+    # @param inputs [Array<String>] files or directories
+    # @param outputs [Array<String>] files or directories
+    # @return [Hash] manifest
+    def self.record(inputs:, outputs:)
+      {
+        "toolchain" => toolchain,
+        "inputs" => digests(inputs),
+        "outputs" => digests(outputs),
+      }
+    end
+
+    # Write a provenance manifest as canonical JSON.
+    def self.write(path, inputs:, outputs:)
+      FileUtils.mkdir_p(File.dirname(path))
+      manifest = record(inputs: inputs, outputs: outputs)
+      File.write(path, "#{JSON.pretty_generate(manifest)}\n")
+      path
+    end
+
+    # Independently recompute every digest in the manifest.
+    #
+    # @return [Array<String>] problems (empty = verified)
+    def self.verify(path)
+      manifest = JSON.parse(File.read(path))
+      SECTIONS.flat_map do |section|
+        manifest.fetch(section, {}).filter_map do |relative, digest|
+          full = File.expand_path(relative)
+          if !File.exist?(full)
+            "#{section}: missing #{relative}"
+          elsif Digest::SHA256.file(full).hexdigest != digest
+            "#{section}: changed #{relative}"
+          end
+        end
+      end
+    end
+
+    # @param paths [Array<String>] files or directories
+    # @return [Hash{String=>String}] relative path → sha256
+    def self.digests(paths)
+      files = paths.flat_map do |path|
+        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
+      end
+      files.select { |file| File.file?(file) }
+        .map { |file| relative(file) }
+        .uniq.sort
+        .to_h { |file| [file, Digest::SHA256.file(file).hexdigest] }
+    end
+
+    def self.relative(path)
+      Pathname.new(File.expand_path(path))
+        .relative_path_from(Pathname.getwd)
+        .to_s
+    rescue ArgumentError
+      path.to_s
+    end
+
+    private_class_method :optional_versions, :sign_with, :executable?
+  end
+end

data/lib/suma/publication.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "expressir"
+
+module Suma
+  # An application-protocol publication bundle: a lead schema (the AP's
+  # ARM/MIM) plus the transitive closure of the schemas it USEs FROM or
+  # REFERENCEs, resolved against a schema manifest.
+  #
+  # The closure is computed from the parsed EXPRESS interfaces
+  # (Expressir `Schema#interfaces`, `Interface#schema.id`), so the bundle
+  # mirrors what a STEP publication actually ships: one AP volume with
+  # every schema its AP needs.
+  #
+  # @example
+  #   publication = Publication.new(
+  #     lead_id: "ap242_foo_mim",
+  #     schema_manifest_file: "schemas.yaml",
+  #   )
+  #   publication.schemas.each { |s| s.save_exp }
+  class Publication
+    attr_reader :lead_id, :schema_manifest_file, :profile
+
+    # @param lead_id [String] schema id of the publication's lead schema
+    # @param schema_manifest_file [String] path to the schema manifest
+    # @param profile [SchemaProfile, Symbol, String, nil] schema profile
+    def initialize(lead_id:, schema_manifest_file:, profile: nil)
+      @lead_id = lead_id
+      @schema_manifest_file = schema_manifest_file
+      @profile = SchemaProfile.fetch(profile)
+    end
+
+    # The bundle's schemas, lead first, then breadth-first over the
+    # interface closure.
+    #
+    # @param output_path [String] directory the caller will export into
+    # @return [Array<ExpressSchema>]
+    # @raise [Suma::SchemaNotFoundError] when the lead or a dependency is
+    #   absent from the manifest
+    def schemas(output_path: ".")
+      entries = load_entries
+      resolve(lead_id, entries, output_path)
+    end
+
+    private
+
+    def load_entries
+      manifest = Expressir::SchemaManifest.from_file(
+        File.expand_path(schema_manifest_file),
+      )
+      manifest.schemas.to_h do |entry|
+        [entry.id, entry]
+      end
+    end
+
+    def resolve(lead, entries, output_path)
+      closure_ids(lead, entries).map do |id|
+        build_schema(id, entries, output_path)
+      end
+    end
+
+    # Breadth-first ids of the interface closure reachable from +lead+.
+    #
+    # @param lead [String]
+    # @param entries [Hash{String => Expressir::SchemaManifestEntry}]
+    # @return [Array<String>] lead first, then closure order
+    def closure_ids(lead, entries)
+      order = []
+      queue = [lead]
+      until queue.empty?
+        id = queue.shift
+        next if order.include?(id)
+
+        order << id
+        queue.concat(interface_ids(id, entries))
+      end
+      order
+    end
+
+    # Interface (USE FROM / REFERENCE) schema ids declared by one schema.
+    #
+    # @param id [String]
+    # @param entries [Hash{String => Expressir::SchemaManifestEntry}]
+    # @return [Array<String>]
+    def interface_ids(id, entries)
+      schema = build_schema(id, entries, ".")
+      dependency_ids(schema)
+    end
+
+    def build_schema(id, entries, output_path)
+      entry = entries.fetch(id) do
+        raise SchemaNotFoundError,
+              "[suma] schema #{id.inspect} (needed by publication " \
+              "#{lead_id.inspect}) not found in #{schema_manifest_file}"
+      end
+      ExpressSchema.new(id: id, path: entry.path,
+                        output_path: output_path,
+                        is_standalone_file: true,
+                        profile: profile)
+    end
+
+    def dependency_ids(schema)
+      parsed = schema.parsed
+      schema_obj = parsed.schemas.first
+      (schema_obj.interfaces || []).filter_map { |i| i.schema&.id }.uniq
+    end
+  end
+end

data/lib/suma/register_manifest_generator.rb

@@ -42,8 +42,14 @@ module Suma
     # @param ref [String] human-readable reference label
     # @param language_code [String] language for section names
     # @param owner [String] dataset owner organisation
+    # @param profile [SchemaProfile, Symbol, String, nil] schema profile
+    #   governing classification, naming, and category ordering
+    # @param year [Integer, nil] edition year
+    # @param supersedes [String, nil] URN/ref this edition supersedes
+    # @param edition [Integer, nil] edition number (tagged as edition:N)
     def initialize(schema_manifest_file, output_path, urn:, id:, ref:,
-                   language_code: "eng", owner: DEFAULT_OWNER)
+                   language_code: "eng", owner: DEFAULT_OWNER, profile: nil,
+                   year: nil, supersedes: nil, edition: nil)
       @schema_manifest_file = File.expand_path(schema_manifest_file)
       @output_path = output_path
       @urn = Suma::Urn.new(urn)
@@ -51,6 +57,10 @@ module Suma
       @ref = ref
       @language_code = language_code
       @owner = owner
+      @profile = SchemaProfile.fetch(profile)
+      @year = year
+      @supersedes = supersedes
+      @edition = edition
     end
 
     # Generate and write register.yaml.
@@ -118,33 +128,91 @@ module Suma
         owner: @owner,
         languages: [@language_code],
         ordering: DEFAULT_ORDERING,
+        year: @year,
+        supersedes: @supersedes,
+        tags: edition_tags,
         sections: build_sections(schemas),
       )
     end
 
-    # Build a list of top-level sections, one per category, in
-    # SchemaCategory::ALL declaration order. Categories with no
-    # schemas are omitted.
+    # Edition metadata for the register: the dataset tag groups sibling
+    # editions, and the edition number groups nothing on its own.
+    def edition_tags
+      tags = ["dataset:#{@id}"]
+      tags << "edition:#{@edition}" if @edition
+      tags
+    end
+
+    # Build a list of top-level sections, one per category, in the
+    # schema profile's category order. Categories with no schemas are
+    # omitted.
     #
     # @param schemas [Array<Expressir::SchemaManifestEntry>]
     # @return [Array<Glossarist::Section>]
     def build_sections(schemas)
       groups = schemas.group_by do |s|
-        SchemaCategory.for_schema(id: s.id, path: s.path)
+        SchemaCategory.for_schema(id: s.id, path: s.path, profile: @profile)
       end
 
-      SchemaCategory::ALL.filter_map do |category|
+      SchemaCategory.for_profile(@profile).filter_map do |category|
         children = groups[category] || []
         next if children.empty?
 
         Glossarist::Section.new(
           id: category.id,
           names: { @language_code => category.label },
-          children: children.map { |s| build_section(s) },
+          children: section_children(category, children),
+        )
+      end
+    end
+
+    # Application protocols are grouped one level deeper: the schemas of
+    # one AP publication (its `_arm`/`_mim` pair) become the children of
+    # a per-AP parent section, so the register mirrors the publication
+    # bundling.
+    #
+    # @param category [SchemaCategory]
+    # @param schemas [Array<Expressir::SchemaManifestEntry>]
+    # @return [Array<Glossarist::Section>]
+    def section_children(category, schemas)
+      if category == SchemaCategory::APPLICATION_PROTOCOLS
+        build_ap_publication_sections(schemas)
+      else
+        schemas.map { |s| build_section(s) }
+      end
+    end
+
+    # Build one parent section per AP publication, keyed by the AP's
+    # base identity (the schema id with its ARM/MIM suffix stripped).
+    #
+    # @param schemas [Array<Expressir::SchemaManifestEntry>]
+    # @return [Array<Glossarist::Section>]
+    def build_ap_publication_sections(schemas)
+      schemas.group_by { |s| ap_base_id(s.id) }.sort_by { |base, _| base }
+        .map do |base, members|
+        Glossarist::Section.new(
+          id: base,
+          names: {
+            @language_code => "Application Protocol: " \
+                              "#{SchemaNaming.display_name(base, profile: @profile)}",
+          },
+          children: members.sort_by { |s| s.id.downcase }
+            .map { |s| build_section(s) },
         )
       end
     end
 
+    # Strip the ARM/MIM suffix to get the AP publication's base identity.
+    #
+    # @param id [String]
+    # @return [String]
+    def ap_base_id(id)
+      %w[_arm _mim].each do |suffix|
+        return id.delete_suffix(suffix) if id.downcase.end_with?(suffix)
+      end
+      id
+    end
+
     # Build a single leaf section from a schema descriptor.
     #
     # @param schema [Expressir::SchemaManifestEntry]
@@ -154,7 +222,7 @@ module Suma
         id: schema.id,
         names: {
           @language_code => SchemaNaming.prefixed_name(
-            schema.id, path: schema.path
+            schema.id, path: schema.path, profile: @profile
           ),
         },
       )

data/lib/suma/schema_cache.rb

@@ -4,25 +4,45 @@ require "digest"
 require "fileutils"
 
 module Suma
-  # Content-addressed, on-disk cache of *generated* EXPRESS schema output --
-  # the plain (annotations stripped, ISO copyright kept) or annotated `.exp`
-  # text produced from a source schema.
+  # Content-addressed, on-disk cache of build artifacts derived from EXPRESS
+  # source text.
   #
-  # The cache key composes the source bytes, the Expressir version, and the
-  # annotation mode. Those three fully determine the output (Expressir's
-  # serialiser is deterministic), so a hit is guaranteed byte-identical to a
-  # fresh generation -- and the store can be shared across builds and git
-  # worktrees: identical source content hits regardless of which build wrote it.
+  # Two artifact kinds share one store:
   #
-  # Supersedes the parsed-AST `Marshal` approach of metanorma/suma#35, which was
-  # brittle across Ruby/Expressir versions and cached the wrong artifact (the
-  # parse, not the output).
+  # - generated `.exp` output (plain or annotated), keyed by source bytes,
+  #   Expressir version, and annotation mode;
+  # - parsed Expressir repositories (S6.1), keyed by source bytes, Expressir
+  #   version, Suma version, and Ruby's Marshal format version.
+  #
+  # Those inputs fully determine the artifact, so a hit is equivalent to a
+  # fresh build and the store can be shared across builds and worktrees.
+  # Parsed entries are Marshal payloads: only point the cache at a directory
+  # this process controls.
   class SchemaCache
+    # Bump when the persisted parsed-repository contract changes.
+    PARSED_FORMAT_VERSION = 1
+
+    attr_reader :hits, :misses
+
+    # Build the configured cache, or a null cache when caching is disabled.
+    #
+    # @param options [Hash] may carry :cache_dir
+    # @return [SchemaCache, NullCache]
+    def self.build(options = {})
+      directory = options[:cache_dir] || ENV.fetch("SUMA_SCHEMA_CACHE_DIR", nil)
+      return NullCache.new if directory.nil? || directory.empty?
+
+      new(directory)
+    end
+
     def initialize(directory)
       @directory = directory.to_s
+      @hits = 0
+      @misses = 0
+      @counter_lock = Mutex.new
     end
 
-    # The cached output for +source+, or +nil+ on a miss.
+    # The cached EXPRESS output for +source+, or +nil+ on a miss.
     def fetch(source, annotations:)
       path = entry_path(source, annotations: annotations)
       return unless File.exist?(path)
@@ -38,26 +58,65 @@ module Suma
       path
     end
 
+    # The cached parsed repository for +source+, or +nil+ on a miss.
+    # Hit and miss counts support the "second build parses nothing" gate.
+    def fetch_parsed(source)
+      path = parsed_entry_path(source)
+      unless File.exist?(path)
+        @counter_lock.synchronize { @misses += 1 }
+        return nil
+      end
+
+      @counter_lock.synchronize { @hits += 1 }
+      # Local build cache only; never point SchemaCache at an untrusted path.
+      Marshal.load(File.binread(path)) # rubocop:disable Security/MarshalLoad
+    end
+
+    # Cache a parsed Expressir repository for +source+.
+    def store_parsed(source, repository)
+      path = parsed_entry_path(source)
+      FileUtils.mkdir_p(@directory)
+      write_atomically(path, Marshal.dump(repository))
+      path
+    end
+
     private
 
     def entry_path(source, annotations:)
       File.join(@directory, key(source, annotations: annotations))
     end
 
+    def parsed_entry_path(source)
+      File.join(@directory, parsed_key(source))
+    end
+
     def key(source, annotations:)
       mode = annotations ? "annotated" : "plain"
       "#{Digest::SHA256.hexdigest(source)}-#{expressir_version}-#{mode}.exp"
     end
 
+    def parsed_key(source)
+      "#{Digest::SHA256.hexdigest(source)}-#{expressir_version}-" \
+        "#{model_version}-#{marshal_version}.parsed"
+    end
+
     def expressir_version
       @expressir_version ||= Expressir::VERSION
     end
 
+    def model_version
+      @model_version ||= "#{Suma::VERSION}-#{PARSED_FORMAT_VERSION}"
+    end
+
+    def marshal_version
+      @marshal_version ||= "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}"
+    end
+
     # Write through a sibling temp file and rename, so a concurrent reader never
     # observes a half-written entry.
     def write_atomically(path, content)
       temp = "#{path}.#{Process.pid}.tmp"
-      File.write(temp, content, encoding: "UTF-8")
+      File.binwrite(temp, content)
       File.rename(temp, path)
     ensure
       File.delete(temp) if temp && File.exist?(temp)

data/lib/suma/schema_category.rb

@@ -34,6 +34,13 @@ module Suma
       types: [ExpressSchema::Type::RESOURCE].freeze,
       directory: "resources",
     )
+    APPLICATION_PROTOCOLS = new(
+      id: "application_protocols",
+      label: "Application Protocols",
+      prefix: "Application Protocol",
+      types: [ExpressSchema::Type::APPLICATION_PROTOCOL].freeze,
+      directory: "applications",
+    )
     MODULES = new(
       id: "modules",
       label: "Application Modules",
@@ -65,18 +72,40 @@ module Suma
 
     ALL = [
       RESOURCES,
+      APPLICATION_PROTOCOLS,
       MODULES,
       BUSINESS_OBJECT_MODELS,
       CORE_MODEL,
       OTHER,
     ].freeze
 
+    BY_ID = ALL.to_h do |category|
+      [category.id, category]
+    end.freeze
+
     def self.for_type(type)
       ALL.find { |category| category.member?(type) } || OTHER
     end
 
-    def self.for_schema(id:, path:)
-      type = ExpressSchema::Type.classify(id: id, path: path)
+    # Look up a category by its slug; unknown ids fall back to OTHER so
+    # that schema profiles naming an unknown category still resolve.
+    #
+    # @param id [String, Symbol]
+    # @return [SchemaCategory]
+    def self.find(id)
+      BY_ID.fetch(id.to_s, OTHER)
+    end
+
+    # Ordered category list for a schema profile.
+    #
+    # @param profile [SchemaProfile, Symbol, String, nil]
+    # @return [Array<SchemaCategory>]
+    def self.for_profile(profile)
+      SchemaProfile.fetch(profile).category_ids.map { |id| find(id) }
+    end
+
+    def self.for_schema(id:, path:, profile: nil)
+      type = ExpressSchema::Type.classify(id: id, path: path, profile: profile)
       for_type(type)
     end
   end

data/lib/suma/schema_collection.rb

@@ -6,6 +6,7 @@ module Suma
   class SchemaCollection
     attr_accessor :config, :schemas, :docs, :output_path_docs,
                   :output_path_schemas, :manifest
+    attr_reader :cache
 
     def initialize(config: nil, config_yaml: nil, output_path_docs: nil, # rubocop:disable Metrics/ParameterLists
                    output_path_schemas: nil, manifest: nil,
@@ -21,6 +22,7 @@ module Suma
       @config ||= config_yaml && Expressir::SchemaManifest.from_file(config_yaml)
       @manifest = manifest
       @options = options
+      @cache = SchemaCache.build(@options)
     end
 
     def doc_from_schema_name(schema_name)
@@ -40,21 +42,71 @@ module Suma
 
     def compile
       finalize
+      targets = rebuild_targets
 
       exporter = SchemaExporter.new(
-        schemas: schemas.values,
+        schemas: targets ? schemas_for(targets) : schemas.values,
         output_path: @output_path_schemas,
         options: @options,
       )
       exporter.export
 
-      docs.each_pair do |_schema_id, compiler|
-        compiler.compile
-      end
+      docs_for(targets).each_value(&:compile)
+      persist_build_state
+    end
+
+    # Schema ids an incremental build must re-render: changed schemas plus
+    # their transitive dependents. Returns nil when a full build is required
+    # (incremental disabled, outputs missing, or no prior state).
+    #
+    # Requires {#finalize} to have populated the collection.
+    #
+    # @return [Array<String>, nil]
+    def rebuild_targets
+      return nil unless @options[:incremental]
+      return nil unless outputs_present?
+
+      plan.rebuild
     end
 
     private
 
+    def plan
+      @plan ||= BuildPlan.new(
+        entries: @config.schemas,
+        graph: CorpusGraph.new(schemas.values.map(&:parsed)),
+        previous_state: BuildPlan.read_state(state_file),
+      )
+    end
+
+    def state_file
+      @options[:state_file] ||
+        File.join(@output_path_schemas, ".suma-build-state.json")
+    end
+
+    def outputs_present?
+      File.directory?(@output_path_docs) &&
+        File.directory?(@output_path_schemas)
+    end
+
+    def persist_build_state
+      return unless @options[:incremental]
+
+      BuildPlan.write_state(state_file, plan.current_state)
+    end
+
+    def schemas_for(targets)
+      wanted = targets.map(&:downcase)
+      schemas.values.select { |schema| wanted.include?(schema.id.downcase) }
+    end
+
+    def docs_for(targets)
+      return docs unless targets
+
+      wanted = targets.map(&:downcase)
+      docs.select { |id, _compiler| wanted.include?(id.downcase) }
+    end
+
     def process_schemas(schemas, template_class)
       schemas.each { |s| process_schema(s, template_class) }
     end
@@ -65,7 +117,8 @@ module Suma
       )
       express = ExpressSchema.new(
         id: config_schema.id, path: config_schema.path.to_s,
-        output_path: @output_path_schemas.join(category.directory).to_s
+        output_path: @output_path_schemas.join(category.directory).to_s,
+        cache: @cache
       )
 
       compiler = SchemaCompiler.new(