suma 0.5.8 → 0.5.10

data/lib/suma/provenance.rb

@@ -83,6 +83,9 @@ module Suma
       end
     end
 
+    # Gems whose versions determine rendered document bytes.
+    RENDER_GEM_PREFIXES = %w[metanorma- isodoc].freeze
+
     # @return [Hash] toolchain versions that determine output bytes
     def self.toolchain
       {
@@ -98,30 +101,63 @@ module Suma
       versions
     end
 
+    # Toolchain identity for cached rendered documents: the provenance
+    # toolchain plus every installed Metanorma/Isodoc gem, so a renderer or
+    # flavor update invalidates cached renders even when source is unchanged.
+    def self.render_toolchain
+      @render_toolchain ||= toolchain.merge("render" => render_gems)
+    end
+
+    def self.render_gems
+      gem_specs.select { |spec| render_gem?(spec.name) }
+        .sort_by { |spec| [spec.name, spec.version.to_s] }
+        .map { |spec| "#{spec.name}-#{spec.version}" }
+    end
+
+    def self.render_gem?(name)
+      name == "metanorma" || RENDER_GEM_PREFIXES.any? do |prefix|
+        name.start_with?(prefix)
+      end
+    end
+
+    def self.gem_specs
+      Gem::Specification.select { |spec| spec.name.match?(/\A(?:metanorma|isodoc)/) }
+    end
+
     # @param inputs [Array<String>] files or directories
     # @param outputs [Array<String>] files or directories
+    # @param exclude [Array<String>] files or directories omitted from the
+    #   manifest and from the unrecorded-output check
     # @return [Hash] manifest
-    def self.record(inputs:, outputs:)
+    def self.record(inputs:, outputs:, exclude: [])
+      excluded = exclude.map { |path| File.expand_path(path) }
       {
         "toolchain" => toolchain,
-        "inputs" => digests(inputs),
-        "outputs" => digests(outputs),
+        "inputs" => digests(inputs, exclude: excluded),
+        "outputs" => digests(outputs, exclude: excluded),
+        "output_roots" => output_roots(outputs, exclude: excluded),
+        "excluded" => excluded.map { |path| relative(path) }.sort,
       }
     end
 
     # Write a provenance manifest as canonical JSON.
-    def self.write(path, inputs:, outputs:)
+    def self.write(path, inputs:, outputs:, exclude: [])
       FileUtils.mkdir_p(File.dirname(path))
-      manifest = record(inputs: inputs, outputs: outputs)
+      manifest = record(inputs: inputs, outputs: outputs, exclude: exclude)
       File.write(path, "#{JSON.pretty_generate(manifest)}\n")
       path
     end
 
-    # Independently recompute every digest in the manifest.
+    # Independently recompute every digest in the manifest and flag files
+    # that appeared under a recorded output root but are not in the manifest.
     #
     # @return [Array<String>] problems (empty = verified)
     def self.verify(path)
       manifest = JSON.parse(File.read(path))
+      digest_problems(manifest) + unrecorded_outputs(manifest)
+    end
+
+    def self.digest_problems(manifest)
       SECTIONS.flat_map do |section|
         manifest.fetch(section, {}).filter_map do |relative, digest|
           full = File.expand_path(relative)
@@ -134,18 +170,59 @@ module Suma
       end
     end
 
+    def self.unrecorded_outputs(manifest)
+      recorded = manifest.fetch("outputs", {})
+      excluded = expanded(manifest["excluded"])
+      Array(manifest["output_roots"]).flat_map do |root|
+        output_files(root)
+          .reject { |file| recorded.key?(file) || excluded?(file, excluded) }
+          .map { |file| "outputs: unrecorded #{file}" }
+      end.sort
+    end
+
+    def self.output_files(root)
+      full_root = File.expand_path(root)
+      return [] unless File.directory?(full_root)
+
+      files([full_root]).map { |file| relative(file) }.uniq
+    end
+
+    def self.expanded(paths)
+      Array(paths).map { |path| File.expand_path(path) }
+    end
+
+    def self.excluded?(relative_path, excluded)
+      full = File.expand_path(relative_path)
+      excluded.any? do |path|
+        full == path || full.start_with?("#{path}#{File::SEPARATOR}")
+      end
+    end
+
     # @param paths [Array<String>] files or directories
+    # @param exclude [Array<String>] expanded paths to skip
     # @return [Hash{String=>String}] relative path → sha256
-    def self.digests(paths)
-      files = paths.flat_map do |path|
-        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
-      end
-      files.select { |file| File.file?(file) }
+    def self.digests(paths, exclude: [])
+      files(paths)
+        .reject { |file| excluded?(file, exclude) }
         .map { |file| relative(file) }
         .uniq.sort
         .to_h { |file| [file, Digest::SHA256.file(file).hexdigest] }
     end
 
+    def self.files(paths)
+      paths.flat_map do |path|
+        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
+      end.select { |file| File.file?(file) }
+    end
+
+    # Relative paths of the directory entries among +outputs+.
+    def self.output_roots(outputs, exclude: [])
+      outputs.select { |path| File.directory?(path) }
+        .map { |path| relative(path) }
+        .reject { |path| excluded?(path, exclude) }
+        .uniq.sort
+    end
+
     def self.relative(path)
       Pathname.new(File.expand_path(path))
         .relative_path_from(Pathname.getwd)
@@ -154,6 +231,9 @@ module Suma
       path.to_s
     end
 
-    private_class_method :optional_versions, :sign_with, :executable?
+    private_class_method :optional_versions, :sign_with, :executable?,
+                         :render_gems, :render_gem?, :gem_specs,
+                         :digest_problems, :unrecorded_outputs, :output_files,
+                         :expanded, :excluded?, :files
   end
 end

data/lib/suma/schema_collection.rb

@@ -125,6 +125,7 @@ module Suma
         schema: express,
         output_path: @output_path_docs.join(express.id),
         template: template_class.new(express.id),
+        no_valid: @options[:no_valid],
       )
 
       @docs[express.id] = compiler

data/lib/suma/schema_compiler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
 require "fileutils"
+require "json"
 require "pathname"
 
 module Suma
@@ -12,20 +14,32 @@ module Suma
   # that the caller injects. This split means templates can be tested as pure
   # functions and the compiler can be tested with a real adoc fixture, without
   # either depending on the other.
+  #
+  # Rendering is skipped when the adoc, the schema config, the schema source,
+  # and the rendering toolchain are unchanged since the outputs were written:
+  # the render key is stored beside the outputs.
   class SchemaCompiler
+    RENDER_KEY_FILE = ".suma-render-key"
+
     attr_reader :schema, :id, :output_path, :template
 
-    def initialize(schema:, output_path:, template:)
+    def initialize(schema:, output_path:, template:, no_valid: false)
       @schema = schema
       @id = schema.id
       @output_path = output_path
       @template = template
+      @no_valid = no_valid
     end
 
     def compile
       save_config
-      save_adoc
+      adoc = template.render(config_relative_path)
+      save_adoc(adoc)
+      key = render_key(adoc)
+      return reuse_render if rendered_current?(key)
+
       invoke_metanorma
+      write_render_key(key)
       self
     end
 
@@ -37,6 +51,13 @@ module Suma
       template.extensions
     end
 
+    # Options handed to Metanorma::Compile. Exposed for specs.
+    def compile_options
+      { agree_to_terms: true, install_fonts: false }.tap do |options|
+        options[:novalid] = true if @no_valid
+      end
+    end
+
     private
 
     def filename_adoc(ext = "adoc")
@@ -47,15 +68,20 @@ module Suma
       File.join(output_path, "schema_#{id}.yaml")
     end
 
-    def save_adoc
-      log_relative "Save EXPRESS adoc", filename_adoc
-
-      FileUtils.mkdir_p(File.dirname(filename_adoc))
+    def render_key_path
+      File.join(output_path, RENDER_KEY_FILE)
+    end
 
-      config_relative = Pathname.new(filename_config)
+    def config_relative_path
+      Pathname.new(filename_config)
         .relative_path_from(Pathname.new(File.dirname(filename_adoc)))
+    end
+
+    def save_adoc(content)
+      log_relative "Save EXPRESS adoc", filename_adoc
 
-      File.write(filename_adoc, template.render(config_relative))
+      FileUtils.mkdir_p(File.dirname(filename_adoc))
+      File.write(filename_adoc, content)
     end
 
     def save_config
@@ -68,16 +94,56 @@ module Suma
       config.save_to_path(filename_config)
     end
 
+    # Everything the rendered document depends on. The schema source is an
+    # input even though the adoc references it only by path.
+    def render_key(adoc)
+      Digest::SHA256.hexdigest(
+        [adoc, File.read(filename_config), schema_source_digest,
+         JSON.generate(Provenance.render_toolchain)].join("\0"),
+      )
+    end
+
+    def schema_source_digest
+      path = schema.path.to_s
+      File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing:#{path}"
+    end
+
+    def rendered_current?(key)
+      File.file?(render_key_path) && File.read(render_key_path) == key &&
+        expected_output_paths.all? { |path| File.file?(path) }
+    end
+
+    def expected_output_paths
+      extensions.split(",").map { |ext| filename_adoc(ext.strip) }
+    end
+
+    def write_render_key(key)
+      File.write(render_key_path, key)
+    end
+
+    def reuse_render
+      log_relative "Reusing rendered schema doc (id: #{id})", filename_adoc
+      self
+    end
+
     def invoke_metanorma
       log_relative "Compiling schema (id: #{id})", filename_adoc
-      Metanorma::Compile.new.compile(
-        filename_adoc,
-        agree_to_terms: true,
-        install_fonts: false,
-      )
+      Metanorma::Compile.new.compile(filename_adoc, **compile_options)
+      verify_outputs!
       log_relative "Compiling schema (id: #{id}) ... done!", filename_adoc
     end
 
+    # Metanorma::Compile#compile can return nil for a handled failure instead
+    # of raising. Never cache a render that produced no output.
+    def verify_outputs!
+      missing = expected_output_paths.reject { |path| File.file?(path) }
+      return if missing.empty?
+
+      names = missing.map { |path| File.basename(path) }.join(", ")
+      raise CompilationError,
+            "Metanorma produced no #{names} for schema #{id}"
+    end
+
     def log_relative(prefix, path)
       relative = Pathname.new(path).relative_path_from(Dir.pwd)
       Utils.log "#{prefix}: #{relative}"

data/lib/suma/schema_manifest_generator.rb

@@ -48,11 +48,18 @@ exclude_paths: nil)
       YAML.safe_load(File.read(file_path, encoding: "UTF-8"), aliases: true)
     end
 
+    # Collection manifest files referenced by the site manifest. Both
+    # manifest shapes are handled: legacy `docref` entries and the
+    # nested `documents` entries (metanorma 2.x collection manifests).
+    # Only `file` references count — `fileref` entries are attachments
+    # (documents, schema exports), not collection manifests.
     def load_manifest_files(collection_files)
-      collection_files.map do |c|
+      collection_files.flat_map do |c|
         collection_data = load_yaml(c)
-        collection_data["manifest"]["docref"].map { |docref| docref["file"] }
-      end.flatten
+        manifest = collection_data["manifest"] || {}
+        entries = Array(manifest["docref"]) + Array(manifest["documents"])
+        entries.filter_map { |entry| entry["file"] }
+      end
     end
 
     def load_project_schemas(manifest_files)

data/lib/suma/schema_profile.rb

@@ -126,9 +126,21 @@ module Suma
       category_ids: %w[other],
     )
 
+    # buildingSMART IFC: one standalone schema per edition (IFC4, IFC4X3,
+    # ...), entities prefixed Ifc, no ARM/MIM-style decomposition.
+    IFC = new(
+      id: "ifc",
+      rules: [],
+      suffix_labels: {},
+      acronyms: %w[bim ifc idf 2d 3d 4d 5d],
+      acronym_patterns: [/\Aifc\d+(x\d+)*\z/],
+      category_ids: %w[other],
+    )
+
     REGISTRY = {
       "iso10303" => ISO_10303,
       "generic" => GENERIC,
+      "ifc" => IFC,
     }.freeze
   end
 end

data/lib/suma/schema_template/anchors.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Suma
+  module SchemaTemplate
+    # The Liquid anchor block shared by the schema document templates: one
+    # [[id]] for every element an Expressir cross-reference may target, so
+    # every xref the hyperlinked formatter emits resolves in the rendered
+    # document. Expressir links schema-qualified element ids
+    # (`schema.entity`) but bare `entity.attribute` pairs, hence the two
+    # bookmark forms.
+    module Anchors
+      # Xref targets are always qualified by the schema's *declared* id, so
+      # anchors are too — independent of the manifest key's casing.
+      SCHEMA_REF = "{{ schema.id }}"
+
+      # The raw anchor block contains Liquid comments and newlines that are
+      # illegal on the section header line; strip them before interpolating.
+      def rendered_anchors
+        anchor_block
+          .gsub(%r{//[^\r\n]+}, "")
+          .gsub(/[\n\r]+/, "")
+          .gsub(/^[\n\r]/, "")
+      end
+
+      private
+
+      def anchor_block
+        <<~HEREDOC
+          // _fund_cons.liquid
+          [[#{schema_id}_funds]]
+
+          #{constants_and_types_anchors}
+
+          #{entity_anchors}
+
+          #{declaration_anchors}
+        HEREDOC
+      end
+
+      def constants_and_types_anchors
+        <<~HEREDOC
+          // _constants.liquid
+          {% if schema.constants.size > 0 %}
+          #{bookmark('constants')}
+          {% for thing in schema.constants %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _types.liquid
+          {% if schema.types.size > 0 %}
+          #{bookmark('types')}
+          // _type.liquid
+          {% for thing in schema.types %}
+          #{bookmark('{{thing.id}}')}
+          {% if thing.items.size > 0 %}
+          // _type_items.liquid
+          #{bookmark('{{thing.id}}.items')}
+          {% for item in thing.items %}
+          #{bookmark('{{thing.id}}.items.{{item.id}}')}
+          {% endfor %}
+          {% endif %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def entity_anchors
+        <<~HEREDOC
+          // _entities.liquid
+          {% if schema.entities.size > 0 %}
+          #{bookmark('entities')}
+          // _entity.liquid
+          {% for thing in schema.entities %}
+          #{bookmark('{{thing.id}}')}
+          {% for attr in thing.attributes %}
+          #{bookmark_local('{{thing.id}}.{{attr.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def declaration_anchors
+        <<~HEREDOC
+          // _subtype_constraints.liquid
+          {% if schema.subtype_constraints.size > 0 %}
+          #{bookmark('subtype_constraints')}
+          // _subtype_constraint.liquid
+          {% for thing in schema.subtype_constraints %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _functions.liquid
+          {% if schema.functions.size > 0 %}
+          #{bookmark('functions')}
+          // _function.liquid
+          {% for thing in schema.functions %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _procedures.liquid
+          {% if schema.procedures.size > 0 %}
+          #{bookmark('procedures')}
+          // _procedure.liquid
+          {% for thing in schema.procedures %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _rules.liquid
+          {% if schema.rules.size > 0 %}
+          #{bookmark('rules')}
+          // _rule.liquid
+          {% for thing in schema.rules %}
+          #{bookmark('{{thing.id}}')}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      # Functions and procedures carry parameters and local variables, and
+      # Expressir links references to them as bare `owner.name` pairs.
+      def callable_anchors
+        <<~HEREDOC
+          #{bookmark('{{thing.id}}')}
+          {% for param in thing.parameters %}
+          #{bookmark_local('{{thing.id}}.{{param.id}}')}
+          {% endfor %}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+        HEREDOC
+      end
+
+      def bookmark(anchor)
+        "[[#{mangled(SCHEMA_REF)}.#{mangled(anchor)}]]"
+      end
+
+      def bookmark_local(anchor)
+        "[[#{mangled(anchor)}]]"
+      end
+
+      def mangled(anchor)
+        anchor.gsub("}}", ' | replace: "\", "-", "-}}')
+      end
+    end
+  end
+end

data/lib/suma/schema_template/document.rb

@@ -4,10 +4,11 @@ module Suma
   module SchemaTemplate
     # Emits an AsciiDoc body with cross-reference anchors for every
     # schema element so other documents can deep-link into the compiled
-    # HTML. Only XML is produced — the anchors only resolve against the
+    # output. Only XML is produced — the anchors only resolve against the
     # XML output, not the HTML rendering.
     class Document
       EXTENSIONS = "xml"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -44,98 +45,6 @@ module Suma
 
         ADOC
       end
-
-      private
-
-      # The raw anchor block contains Liquid comments and newlines that
-      # are illegal on the section header line; strip them before
-      # interpolating.
-      def rendered_anchors
-        schema_anchors.gsub(%r{//[^\r\n]+}, "").gsub(/[\n\r]+/, "").gsub(
-          /^[\n\r]/, ""
-        )
-      end
-
-      def schema_anchors
-        <<~HEREDOC
-          // _fund_cons.liquid
-          [[#{schema_id}_funds]]
-
-          // _constants.liquid
-          {% if schema.constants.size > 0 %}
-          #{bookmark('constants')}
-          {% for thing in schema.constants %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _types.liquid
-          {% if schema.types.size > 0 %}
-          #{bookmark('types')}
-          // _type.liquid
-          {% for thing in schema.types %}
-          #{bookmark('{{thing.id}}')}
-          {% if thing.items.size > 0 %}
-          // _type_items.liquid
-          #{bookmark('{{thing.id}}.items')}
-          {% for item in thing.items %}
-          #{bookmark('{{thing.id}}.items.{{item.id}}')}
-          {% endfor %}
-          {% endif %}
-          {% endfor %}
-          {% endif %}
-
-          // _entities.liquid
-          {% if schema.entities.size > 0 %}
-          #{bookmark('entities')}
-          {% for thing in schema.entities %}
-          // _entity.liquid
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _subtype_constraints.liquid
-          {% if schema.subtype_constraints.size > 0 %}
-          #{bookmark('subtype_constraints')}
-          // _subtype_constraint.liquid
-          {% for thing in schema.subtype_constraints %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _functions.liquid
-          {% if schema.functions.size > 0 %}
-          #{bookmark('functions')}
-          // _function.liquid
-          {% for thing in schema.functions %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _procedures.liquid
-          {% if schema.procedures.size > 0 %}
-          #{bookmark('procedures')}
-          // _procedure.liquid
-          {% for thing in schema.procedures %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _rules.liquid
-          {% if schema.rules.size > 0 %}
-          #{bookmark('rules')}
-          // _rule.liquid
-          {% for thing in schema.rules %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-        HEREDOC
-      end
-
-      def bookmark(anchor)
-        mangled = anchor.gsub("}}", ' | replace: "\", "-", "-}}')
-        "[[#{schema_id}.#{mangled}]]"
-      end
     end
   end
 end

data/lib/suma/schema_template/plain.rb

@@ -2,10 +2,13 @@
 
 module Suma
   module SchemaTemplate
-    # Emits a plain AsciiDoc body for a single EXPRESS schema, producing
-    # both HTML and XML outputs via Metanorma.
+    # Emits an anchored, hyperlinked AsciiDoc body for a single EXPRESS
+    # schema, producing both HTML and XML outputs via Metanorma. Every
+    # element cross-reference the hyperlinked source emits resolves inside
+    # the rendered document through the shared anchor block.
     class Plain
       EXTENSIONS = "xml,html"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -29,12 +32,13 @@ module Suma
           ----
           {% for schema in context.schemas %}
 
-          [%unnumbered]
-          == #{schema_id}
+          [[#{schema_id}]]
+          [%unnumbered,type=express]
+          == #{schema_id} #{rendered_anchors}
 
-          [source%unnumbered]
+          [source%unnumbered,subs="+macros"]
           --
-          {{ schema.formatted }}
+          {{ schema.formatted_hyperlinked_adoc }}
           --
           {% endfor %}
           ----

data/lib/suma/schema_template.rb

@@ -13,6 +13,7 @@ module Suma
   #
   # Composition with the compiler lives in SchemaCompiler.
   module SchemaTemplate
+    autoload :Anchors, "suma/schema_template/anchors"
     autoload :Plain, "suma/schema_template/plain"
     autoload :Document, "suma/schema_template/document"
   end