suma 0.5.9 → 0.5.10

data/lib/suma/schema_compiler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
 require "fileutils"
+require "json"
 require "pathname"
 
 module Suma
@@ -12,20 +14,32 @@ module Suma
   # that the caller injects. This split means templates can be tested as pure
   # functions and the compiler can be tested with a real adoc fixture, without
   # either depending on the other.
+  #
+  # Rendering is skipped when the adoc, the schema config, the schema source,
+  # and the rendering toolchain are unchanged since the outputs were written:
+  # the render key is stored beside the outputs.
   class SchemaCompiler
+    RENDER_KEY_FILE = ".suma-render-key"
+
     attr_reader :schema, :id, :output_path, :template
 
-    def initialize(schema:, output_path:, template:)
+    def initialize(schema:, output_path:, template:, no_valid: false)
       @schema = schema
       @id = schema.id
       @output_path = output_path
       @template = template
+      @no_valid = no_valid
     end
 
     def compile
       save_config
-      save_adoc
+      adoc = template.render(config_relative_path)
+      save_adoc(adoc)
+      key = render_key(adoc)
+      return reuse_render if rendered_current?(key)
+
       invoke_metanorma
+      write_render_key(key)
       self
     end
 
@@ -37,6 +51,13 @@ module Suma
       template.extensions
     end
 
+    # Options handed to Metanorma::Compile. Exposed for specs.
+    def compile_options
+      { agree_to_terms: true, install_fonts: false }.tap do |options|
+        options[:novalid] = true if @no_valid
+      end
+    end
+
     private
 
     def filename_adoc(ext = "adoc")
@@ -47,15 +68,20 @@ module Suma
       File.join(output_path, "schema_#{id}.yaml")
     end
 
-    def save_adoc
-      log_relative "Save EXPRESS adoc", filename_adoc
-
-      FileUtils.mkdir_p(File.dirname(filename_adoc))
+    def render_key_path
+      File.join(output_path, RENDER_KEY_FILE)
+    end
 
-      config_relative = Pathname.new(filename_config)
+    def config_relative_path
+      Pathname.new(filename_config)
         .relative_path_from(Pathname.new(File.dirname(filename_adoc)))
+    end
+
+    def save_adoc(content)
+      log_relative "Save EXPRESS adoc", filename_adoc
 
-      File.write(filename_adoc, template.render(config_relative))
+      FileUtils.mkdir_p(File.dirname(filename_adoc))
+      File.write(filename_adoc, content)
     end
 
     def save_config
@@ -68,16 +94,56 @@ module Suma
       config.save_to_path(filename_config)
     end
 
+    # Everything the rendered document depends on. The schema source is an
+    # input even though the adoc references it only by path.
+    def render_key(adoc)
+      Digest::SHA256.hexdigest(
+        [adoc, File.read(filename_config), schema_source_digest,
+         JSON.generate(Provenance.render_toolchain)].join("\0"),
+      )
+    end
+
+    def schema_source_digest
+      path = schema.path.to_s
+      File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing:#{path}"
+    end
+
+    def rendered_current?(key)
+      File.file?(render_key_path) && File.read(render_key_path) == key &&
+        expected_output_paths.all? { |path| File.file?(path) }
+    end
+
+    def expected_output_paths
+      extensions.split(",").map { |ext| filename_adoc(ext.strip) }
+    end
+
+    def write_render_key(key)
+      File.write(render_key_path, key)
+    end
+
+    def reuse_render
+      log_relative "Reusing rendered schema doc (id: #{id})", filename_adoc
+      self
+    end
+
     def invoke_metanorma
       log_relative "Compiling schema (id: #{id})", filename_adoc
-      Metanorma::Compile.new.compile(
-        filename_adoc,
-        agree_to_terms: true,
-        install_fonts: false,
-      )
+      Metanorma::Compile.new.compile(filename_adoc, **compile_options)
+      verify_outputs!
       log_relative "Compiling schema (id: #{id}) ... done!", filename_adoc
     end
 
+    # Metanorma::Compile#compile can return nil for a handled failure instead
+    # of raising. Never cache a render that produced no output.
+    def verify_outputs!
+      missing = expected_output_paths.reject { |path| File.file?(path) }
+      return if missing.empty?
+
+      names = missing.map { |path| File.basename(path) }.join(", ")
+      raise CompilationError,
+            "Metanorma produced no #{names} for schema #{id}"
+    end
+
     def log_relative(prefix, path)
       relative = Pathname.new(path).relative_path_from(Dir.pwd)
       Utils.log "#{prefix}: #{relative}"

data/lib/suma/schema_manifest_generator.rb

@@ -48,11 +48,18 @@ exclude_paths: nil)
       YAML.safe_load(File.read(file_path, encoding: "UTF-8"), aliases: true)
     end
 
+    # Collection manifest files referenced by the site manifest. Both
+    # manifest shapes are handled: legacy `docref` entries and the
+    # nested `documents` entries (metanorma 2.x collection manifests).
+    # Only `file` references count — `fileref` entries are attachments
+    # (documents, schema exports), not collection manifests.
     def load_manifest_files(collection_files)
-      collection_files.map do |c|
+      collection_files.flat_map do |c|
         collection_data = load_yaml(c)
-        collection_data["manifest"]["docref"].map { |docref| docref["file"] }
-      end.flatten
+        manifest = collection_data["manifest"] || {}
+        entries = Array(manifest["docref"]) + Array(manifest["documents"])
+        entries.filter_map { |entry| entry["file"] }
+      end
     end
 
     def load_project_schemas(manifest_files)

data/lib/suma/schema_profile.rb

@@ -126,9 +126,21 @@ module Suma
       category_ids: %w[other],
     )
 
+    # buildingSMART IFC: one standalone schema per edition (IFC4, IFC4X3,
+    # ...), entities prefixed Ifc, no ARM/MIM-style decomposition.
+    IFC = new(
+      id: "ifc",
+      rules: [],
+      suffix_labels: {},
+      acronyms: %w[bim ifc idf 2d 3d 4d 5d],
+      acronym_patterns: [/\Aifc\d+(x\d+)*\z/],
+      category_ids: %w[other],
+    )
+
     REGISTRY = {
       "iso10303" => ISO_10303,
       "generic" => GENERIC,
+      "ifc" => IFC,
     }.freeze
   end
 end

data/lib/suma/schema_template/anchors.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Suma
+  module SchemaTemplate
+    # The Liquid anchor block shared by the schema document templates: one
+    # [[id]] for every element an Expressir cross-reference may target, so
+    # every xref the hyperlinked formatter emits resolves in the rendered
+    # document. Expressir links schema-qualified element ids
+    # (`schema.entity`) but bare `entity.attribute` pairs, hence the two
+    # bookmark forms.
+    module Anchors
+      # Xref targets are always qualified by the schema's *declared* id, so
+      # anchors are too — independent of the manifest key's casing.
+      SCHEMA_REF = "{{ schema.id }}"
+
+      # The raw anchor block contains Liquid comments and newlines that are
+      # illegal on the section header line; strip them before interpolating.
+      def rendered_anchors
+        anchor_block
+          .gsub(%r{//[^\r\n]+}, "")
+          .gsub(/[\n\r]+/, "")
+          .gsub(/^[\n\r]/, "")
+      end
+
+      private
+
+      def anchor_block
+        <<~HEREDOC
+          // _fund_cons.liquid
+          [[#{schema_id}_funds]]
+
+          #{constants_and_types_anchors}
+
+          #{entity_anchors}
+
+          #{declaration_anchors}
+        HEREDOC
+      end
+
+      def constants_and_types_anchors
+        <<~HEREDOC
+          // _constants.liquid
+          {% if schema.constants.size > 0 %}
+          #{bookmark('constants')}
+          {% for thing in schema.constants %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _types.liquid
+          {% if schema.types.size > 0 %}
+          #{bookmark('types')}
+          // _type.liquid
+          {% for thing in schema.types %}
+          #{bookmark('{{thing.id}}')}
+          {% if thing.items.size > 0 %}
+          // _type_items.liquid
+          #{bookmark('{{thing.id}}.items')}
+          {% for item in thing.items %}
+          #{bookmark('{{thing.id}}.items.{{item.id}}')}
+          {% endfor %}
+          {% endif %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def entity_anchors
+        <<~HEREDOC
+          // _entities.liquid
+          {% if schema.entities.size > 0 %}
+          #{bookmark('entities')}
+          // _entity.liquid
+          {% for thing in schema.entities %}
+          #{bookmark('{{thing.id}}')}
+          {% for attr in thing.attributes %}
+          #{bookmark_local('{{thing.id}}.{{attr.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def declaration_anchors
+        <<~HEREDOC
+          // _subtype_constraints.liquid
+          {% if schema.subtype_constraints.size > 0 %}
+          #{bookmark('subtype_constraints')}
+          // _subtype_constraint.liquid
+          {% for thing in schema.subtype_constraints %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _functions.liquid
+          {% if schema.functions.size > 0 %}
+          #{bookmark('functions')}
+          // _function.liquid
+          {% for thing in schema.functions %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _procedures.liquid
+          {% if schema.procedures.size > 0 %}
+          #{bookmark('procedures')}
+          // _procedure.liquid
+          {% for thing in schema.procedures %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _rules.liquid
+          {% if schema.rules.size > 0 %}
+          #{bookmark('rules')}
+          // _rule.liquid
+          {% for thing in schema.rules %}
+          #{bookmark('{{thing.id}}')}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      # Functions and procedures carry parameters and local variables, and
+      # Expressir links references to them as bare `owner.name` pairs.
+      def callable_anchors
+        <<~HEREDOC
+          #{bookmark('{{thing.id}}')}
+          {% for param in thing.parameters %}
+          #{bookmark_local('{{thing.id}}.{{param.id}}')}
+          {% endfor %}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+        HEREDOC
+      end
+
+      def bookmark(anchor)
+        "[[#{mangled(SCHEMA_REF)}.#{mangled(anchor)}]]"
+      end
+
+      def bookmark_local(anchor)
+        "[[#{mangled(anchor)}]]"
+      end
+
+      def mangled(anchor)
+        anchor.gsub("}}", ' | replace: "\", "-", "-}}')
+      end
+    end
+  end
+end

data/lib/suma/schema_template/document.rb

@@ -4,10 +4,11 @@ module Suma
   module SchemaTemplate
     # Emits an AsciiDoc body with cross-reference anchors for every
     # schema element so other documents can deep-link into the compiled
-    # HTML. Only XML is produced — the anchors only resolve against the
+    # output. Only XML is produced — the anchors only resolve against the
     # XML output, not the HTML rendering.
     class Document
       EXTENSIONS = "xml"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -44,98 +45,6 @@ module Suma
 
         ADOC
       end
-
-      private
-
-      # The raw anchor block contains Liquid comments and newlines that
-      # are illegal on the section header line; strip them before
-      # interpolating.
-      def rendered_anchors
-        schema_anchors.gsub(%r{//[^\r\n]+}, "").gsub(/[\n\r]+/, "").gsub(
-          /^[\n\r]/, ""
-        )
-      end
-
-      def schema_anchors
-        <<~HEREDOC
-          // _fund_cons.liquid
-          [[#{schema_id}_funds]]
-
-          // _constants.liquid
-          {% if schema.constants.size > 0 %}
-          #{bookmark('constants')}
-          {% for thing in schema.constants %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _types.liquid
-          {% if schema.types.size > 0 %}
-          #{bookmark('types')}
-          // _type.liquid
-          {% for thing in schema.types %}
-          #{bookmark('{{thing.id}}')}
-          {% if thing.items.size > 0 %}
-          // _type_items.liquid
-          #{bookmark('{{thing.id}}.items')}
-          {% for item in thing.items %}
-          #{bookmark('{{thing.id}}.items.{{item.id}}')}
-          {% endfor %}
-          {% endif %}
-          {% endfor %}
-          {% endif %}
-
-          // _entities.liquid
-          {% if schema.entities.size > 0 %}
-          #{bookmark('entities')}
-          {% for thing in schema.entities %}
-          // _entity.liquid
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _subtype_constraints.liquid
-          {% if schema.subtype_constraints.size > 0 %}
-          #{bookmark('subtype_constraints')}
-          // _subtype_constraint.liquid
-          {% for thing in schema.subtype_constraints %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _functions.liquid
-          {% if schema.functions.size > 0 %}
-          #{bookmark('functions')}
-          // _function.liquid
-          {% for thing in schema.functions %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _procedures.liquid
-          {% if schema.procedures.size > 0 %}
-          #{bookmark('procedures')}
-          // _procedure.liquid
-          {% for thing in schema.procedures %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _rules.liquid
-          {% if schema.rules.size > 0 %}
-          #{bookmark('rules')}
-          // _rule.liquid
-          {% for thing in schema.rules %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-        HEREDOC
-      end
-
-      def bookmark(anchor)
-        mangled = anchor.gsub("}}", ' | replace: "\", "-", "-}}')
-        "[[#{schema_id}.#{mangled}]]"
-      end
     end
   end
 end

data/lib/suma/schema_template/plain.rb

@@ -2,10 +2,13 @@
 
 module Suma
   module SchemaTemplate
-    # Emits a plain AsciiDoc body for a single EXPRESS schema, producing
-    # both HTML and XML outputs via Metanorma.
+    # Emits an anchored, hyperlinked AsciiDoc body for a single EXPRESS
+    # schema, producing both HTML and XML outputs via Metanorma. Every
+    # element cross-reference the hyperlinked source emits resolves inside
+    # the rendered document through the shared anchor block.
     class Plain
       EXTENSIONS = "xml,html"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -29,12 +32,13 @@ module Suma
           ----
           {% for schema in context.schemas %}
 
-          [%unnumbered]
-          == #{schema_id}
+          [[#{schema_id}]]
+          [%unnumbered,type=express]
+          == #{schema_id} #{rendered_anchors}
 
-          [source%unnumbered]
+          [source%unnumbered,subs="+macros"]
           --
-          {{ schema.formatted }}
+          {{ schema.formatted_hyperlinked_adoc }}
           --
           {% endfor %}
           ----

data/lib/suma/schema_template.rb

@@ -13,6 +13,7 @@ module Suma
   #
   # Composition with the compiler lives in SchemaCompiler.
   module SchemaTemplate
+    autoload :Anchors, "suma/schema_template/anchors"
     autoload :Plain, "suma/schema_template/plain"
     autoload :Document, "suma/schema_template/document"
   end

data/lib/suma/site_build.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "fileutils"
+require "pathname"
+
+module Suma
+  # Freshness decision for the rendered collection site.
+  #
+  # The collection render is expensive: Metanorma compiles every member
+  # document. Its output is a pure function of the emitted collection manifest,
+  # the source tree it references, and the rendering toolchain, so this class
+  # keys those inputs, remembers the key and the produced files in the output
+  # directory, and lets the processor skip the render when nothing changed.
+  class SiteBuild
+    STATE_FILE = ".suma-site-build.json"
+
+    # Emitted manifests that the build itself regenerates deterministically
+    # from the sources above; whether they exist yet must not change the key.
+    GENERATED_FILES = %w[collection-output.yaml schemas.yml].freeze
+
+    def initialize(root:, output_directory:, inputs: [])
+      @root = File.expand_path(root)
+      @output_directory = File.expand_path(output_directory)
+      @extra_inputs = inputs.map { |path| File.expand_path(path) }
+    end
+
+    # @return [Array<String>] source files determining the rendered site
+    def source_files
+      @source_files ||= begin
+        files = @extra_inputs.select { |file| File.file?(file) }
+        (files + tree_files).uniq.sort
+      end
+    end
+
+    # @return [String] digest of every source file and the render toolchain
+    def key
+      @key ||= begin
+        digest = Digest::SHA256.new
+        source_files.each do |file|
+          digest << relative(file) << "\0"
+          digest << Digest::SHA256.file(file).hexdigest << "\0"
+        end
+        digest << JSON.generate(Provenance.render_toolchain)
+        digest.hexdigest
+      end
+    end
+
+    # True when the previous render used the same inputs and every file it
+    # produced is still present.
+    def current?
+      state = read_state
+      return false unless state && state["key"] == key
+
+      outputs = Array(state["outputs"])
+      !outputs.empty? &&
+        outputs.all? { |file| File.file?(File.join(@output_directory, file)) }
+    end
+
+    # Persist the key and the produced files for the next build.
+    def record
+      write_state("key" => key, "outputs" => output_files)
+      self
+    end
+
+    def state_path
+      File.join(@output_directory, STATE_FILE)
+    end
+
+    private
+
+    def tree_files
+      excluded = excluded_paths
+      Dir.glob(File.join(@root, "**", "*")).select do |file|
+        File.file?(file) &&
+          excluded.none? { |path| under?(file, path) } &&
+          !generated?(relative(file).tr("\\", "/"))
+      end
+    end
+
+    # Files the collection render writes next to its sources. They are
+    # outputs of the previous build, never inputs, so including them would
+    # change the key on every run.
+    def generated?(relative)
+      base = File.basename(relative)
+      GENERATED_FILES.include?(relative) ||
+        relative.start_with?("relaton/") ||
+        base.start_with?("tmp_document") ||
+        base.end_with?("_index.html", ".err.html", ".log.txt") ||
+        document_artifact?(relative, base)
+    end
+
+    def document_artifact?(relative, base)
+      return false unless relative.start_with?("documents/")
+
+      base == "cover.html" || base == "document.xml" ||
+        (base.start_with?("document.") && base.include?(".xml"))
+    end
+
+    # Generated trees are derived from the sources above and never feed the
+    # site render as inputs of their own. Bundler's install directory is a
+    # dependency artifact: CI conventionally places it at vendor/bundle
+    # inside the project, where it must never count as site source.
+    def excluded_paths
+      fixed = %w[schema_docs plain_schemas .git .bundle vendor/bundle]
+        .map { |dir| File.join(@root, dir) }
+      env = [ENV.fetch("SUMA_SCHEMA_CACHE_DIR", nil),
+             ENV.fetch("BUNDLE_PATH", nil)]
+        .compact
+        .map { |dir| File.expand_path(dir, @root) }
+      [@output_directory] + fixed + env
+    end
+
+    def under?(file, path)
+      file == path || file.start_with?("#{path}#{File::SEPARATOR}")
+    end
+
+    def read_state
+      return unless File.file?(state_path)
+
+      JSON.parse(File.read(state_path))
+    rescue JSON::ParserError
+      nil
+    end
+
+    def write_state(state)
+      FileUtils.mkdir_p(@output_directory)
+      temp = "#{state_path}.#{Process.pid}.tmp"
+      File.write(temp, "#{JSON.pretty_generate(state)}\n")
+      File.rename(temp, state_path)
+    ensure
+      File.delete(temp) if temp && File.exist?(temp)
+    end
+
+    def output_files
+      Dir.glob(File.join(@output_directory, "**", "*"))
+        .select do |file|
+          File.file?(file) && File.basename(file) != STATE_FILE
+        end
+        .map { |file| relative_to(file, @output_directory) }
+        .sort
+    end
+
+    def relative(path)
+      relative_to(path, @root)
+    end
+
+    def relative_to(path, base)
+      Pathname.new(path).relative_path_from(Pathname.new(base)).to_s
+    end
+  end
+end

data/lib/suma/staged_collection_builder.rb

@@ -37,14 +37,16 @@ module Suma
     # +collection_config_path+ is the emitted collection manifest (the same
     # +collection-output.yaml+ the normal build renders). Member filerefs are
     # relative to its directory, so all per-member manifests are written there.
+    # (a data-declaring initializer)
     def initialize(collection_config_path:, output_directory:, coverpage: nil,
-                   store_dir: nil, formats: %i[xml html])
+                   store_dir: nil, formats: %i[xml html], no_valid: false) # rubocop:disable Metrics/ParameterLists
       @config_path = collection_config_path
       @config_dir = File.dirname(File.expand_path(collection_config_path))
       @output_directory = output_directory
       @coverpage = coverpage
       @store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
       @formats = formats
+      @no_valid = no_valid
     end
 
     def build
@@ -97,7 +99,7 @@ module Suma
         manifest_path: one_path,
         opts: { format: [:xml], output_folder: out,
                 preserve_unresolved: true, artifact_store_dir: @store_dir,
-                compile: { install_fonts: false } },
+                compile: compile_flags },
       )
     ensure
       FileUtils.rm_f(one_path)
@@ -118,10 +120,16 @@ module Suma
         manifest_path: reinf_path,
         opts: { format: @formats, output_folder: @output_directory,
                 reinflate: true, coverpage: @coverpage,
-                compile: { install_fonts: false } }.compact,
+                compile: compile_flags }.compact,
       )
     end
 
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
     # Copy each member's stored stub into +dir+ and return docref entries pointing
     # at the copies, keyed by real docid (as the store keys them).
     def stored_docrefs(members, dir)