suma 0.5.9 → 0.5.11

data/lib/suma/processor.rb

@@ -13,8 +13,10 @@ module Suma
 
     def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
                    output_directory: "_site", staged: false,
+                   staged_workers: 1, staged_batch_size: nil,
                    incremental: false, state_file: nil, jobs: 1,
-                   provenance_file: nil, sign_method: nil, sign_key: nil)
+                   provenance_file: nil, sign_method: nil, sign_key: nil,
+                   no_valid: false)
       @metanorma_yaml_path = metanorma_yaml_path
       @schemas_all_path = schemas_all_path
       @compile_flag = compile
@@ -22,12 +24,17 @@ module Suma
       # Opt-in memory-bounded staged build (metanorma/suma#94); the default
       # single-process build below is unchanged when false.
       @staged = staged
+      @staged_workers = staged_workers
+      @staged_batch_size = staged_batch_size
       @incremental = incremental
       @state_file = state_file
       @jobs = jobs
       @provenance_file = provenance_file
       @sign_method = sign_method
       @sign_key = sign_key
+      # Skip Metanorma document validation (metanorma `--no-valid`). Opt-in:
+      # for collections whose flavor's validators are still in flight.
+      @no_valid = no_valid
     end
 
     def run
@@ -39,7 +46,6 @@ module Suma
         Utils.log "Compiling schema collection..."
         compile_schema(schemas_all_path, collection_config)
 
-        Utils.log "Compiling complete collection..."
         compile_collection(collection_config, output_directory)
       end
 
@@ -56,6 +62,7 @@ module Suma
         @provenance_file,
         inputs: provenance_inputs,
         outputs: provenance_outputs,
+        exclude: provenance_excludes,
       )
       if @sign_method
         signature = Provenance.sign(
@@ -80,6 +87,31 @@ module Suma
        @output_directory]
     end
 
+    # Build metadata that is not site content: the site freshness state, the
+    # staged-build store, and a provenance file written inside the output tree.
+    def provenance_excludes
+      excludes = [File.join(@output_directory, SiteBuild::STATE_FILE)]
+      if @staged
+        excludes << File.join(@output_directory,
+                              StagedCollectionBuilder::STORE_DIRNAME)
+        excludes << File.join(@output_directory, "_staged_reinflate")
+      end
+      excludes + provenance_artifacts.select { |path| within_output?(path) }
+    end
+
+    def provenance_artifacts
+      return [] unless @provenance_file
+
+      base = File.expand_path(@provenance_file)
+      [base, "#{base}.minisig", "#{base}.bundle"]
+    end
+
+    def within_output?(path)
+      full = File.expand_path(path)
+      output = File.expand_path(@output_directory)
+      full == output || full.start_with?("#{output}#{File::SEPARATOR}")
+    end
+
     def export_schema_config
       site_config = Suma::SiteConfig::Config.from_file(metanorma_yaml_path)
 
@@ -108,6 +140,7 @@ module Suma
           incremental: @incremental,
           state_file: @state_file,
           jobs: @jobs,
+          no_valid: @no_valid,
         },
       )
 
@@ -115,36 +148,83 @@ module Suma
     end
 
     def compile_collection(collection_config, output_directory)
-      metanorma_collection, collection_opts = build_collection(
-        collection_config, output_directory
-      )
+      prepare_collection_manifest(collection_config)
+      write_collection_manifest(collection_config)
+
+      if site_build.current?
+        Utils.log "Collection outputs are up to date; 0 documents rendered."
+        return
+      end
+
+      Utils.log "Rendering collection site..."
+      render_collection(collection_config, output_directory)
+      canonicalizer.canonicalize_documents
+      site_build.record
+    end
+
+    # Generated manifest identifiers are canonicalized in the model, before
+    # serialization: patching the dumped YAML would preserve Psych's
+    # value-dependent (random) quoting decision.
+    def prepare_collection_manifest(collection_config)
+      ManifestTraverser.new(collection_config.manifest)
+        .remove_schemas_only_sources
+      canonicalizer.canonicalize_manifest(collection_config.manifest)
+    end
 
+    def write_collection_manifest(collection_config)
+      collection_config.to_file(COLLECTION_OUTPUT_PATH)
+    end
+
+    def render_collection(collection_config, output_directory)
       if @staged
-        # build_collection has written the emitted manifest; stage each member in
-        # its own process, then reinflate. Bounds peak memory to a single member.
+        # The emitted manifest is the staging input; each member is compiled
+        # in its own process and then reinflated into +output_directory+.
         StagedCollectionBuilder.new(
           collection_config_path: COLLECTION_OUTPUT_PATH,
           output_directory: output_directory,
-          coverpage: collection_opts[:coverpage],
+          coverpage: collection_coverpage(collection_config),
+          no_valid: @no_valid,
+          workers: @staged_workers,
+          batch_size: @staged_batch_size ||
+            StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
         ).build
       else
-        metanorma_collection.render(collection_opts)
+        Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)
+          .render(collection_options(collection_config, output_directory))
       end
     end
 
-    def build_collection(collection_config, output_directory)
-      new_collection_config_path = COLLECTION_OUTPUT_PATH
-      ManifestTraverser.new(collection_config.manifest).remove_schemas_only_sources
-      collection_config.to_file(new_collection_config_path)
-
-      collection = Metanorma::Collection.parse(new_collection_config_path)
-
-      collection_opts = {
+    def collection_options(collection_config, output_directory)
+      {
         output_folder: output_directory,
-        compile: { install_fonts: false },
-        coverpage: collection_config.coverpage || "cover.html",
+        compile: compile_flags,
+        coverpage: collection_coverpage(collection_config),
       }
-      [collection, collection_opts]
+    end
+
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
+    def collection_coverpage(collection_config)
+      collection_config.coverpage || "cover.html"
+    end
+
+    def site_build
+      @site_build ||= SiteBuild.new(
+        root: File.dirname(File.expand_path(metanorma_yaml_path)),
+        output_directory: output_directory,
+        inputs: [schemas_all_path],
+      )
+    end
+
+    def canonicalizer
+      @canonicalizer ||= OutputCanonicalizer.new(
+        output_directory: output_directory,
+        sources: site_build.source_files,
+      )
     end
   end
 end

data/lib/suma/provenance.rb

@@ -83,6 +83,9 @@ module Suma
       end
     end
 
+    # Gems whose versions determine rendered document bytes.
+    RENDER_GEM_PREFIXES = %w[metanorma- isodoc].freeze
+
     # @return [Hash] toolchain versions that determine output bytes
     def self.toolchain
       {
@@ -98,30 +101,63 @@ module Suma
       versions
     end
 
+    # Toolchain identity for cached rendered documents: the provenance
+    # toolchain plus every installed Metanorma/Isodoc gem, so a renderer or
+    # flavor update invalidates cached renders even when source is unchanged.
+    def self.render_toolchain
+      @render_toolchain ||= toolchain.merge("render" => render_gems)
+    end
+
+    def self.render_gems
+      gem_specs.select { |spec| render_gem?(spec.name) }
+        .sort_by { |spec| [spec.name, spec.version.to_s] }
+        .map { |spec| "#{spec.name}-#{spec.version}" }
+    end
+
+    def self.render_gem?(name)
+      name == "metanorma" || RENDER_GEM_PREFIXES.any? do |prefix|
+        name.start_with?(prefix)
+      end
+    end
+
+    def self.gem_specs
+      Gem::Specification.select { |spec| spec.name.match?(/\A(?:metanorma|isodoc)/) }
+    end
+
     # @param inputs [Array<String>] files or directories
     # @param outputs [Array<String>] files or directories
+    # @param exclude [Array<String>] files or directories omitted from the
+    #   manifest and from the unrecorded-output check
     # @return [Hash] manifest
-    def self.record(inputs:, outputs:)
+    def self.record(inputs:, outputs:, exclude: [])
+      excluded = exclude.map { |path| File.expand_path(path) }
       {
         "toolchain" => toolchain,
-        "inputs" => digests(inputs),
-        "outputs" => digests(outputs),
+        "inputs" => digests(inputs, exclude: excluded),
+        "outputs" => digests(outputs, exclude: excluded),
+        "output_roots" => output_roots(outputs, exclude: excluded),
+        "excluded" => excluded.map { |path| relative(path) }.sort,
       }
     end
 
     # Write a provenance manifest as canonical JSON.
-    def self.write(path, inputs:, outputs:)
+    def self.write(path, inputs:, outputs:, exclude: [])
       FileUtils.mkdir_p(File.dirname(path))
-      manifest = record(inputs: inputs, outputs: outputs)
+      manifest = record(inputs: inputs, outputs: outputs, exclude: exclude)
       File.write(path, "#{JSON.pretty_generate(manifest)}\n")
       path
     end
 
-    # Independently recompute every digest in the manifest.
+    # Independently recompute every digest in the manifest and flag files
+    # that appeared under a recorded output root but are not in the manifest.
     #
     # @return [Array<String>] problems (empty = verified)
     def self.verify(path)
       manifest = JSON.parse(File.read(path))
+      digest_problems(manifest) + unrecorded_outputs(manifest)
+    end
+
+    def self.digest_problems(manifest)
       SECTIONS.flat_map do |section|
         manifest.fetch(section, {}).filter_map do |relative, digest|
           full = File.expand_path(relative)
@@ -134,18 +170,59 @@ module Suma
       end
     end
 
+    def self.unrecorded_outputs(manifest)
+      recorded = manifest.fetch("outputs", {})
+      excluded = expanded(manifest["excluded"])
+      Array(manifest["output_roots"]).flat_map do |root|
+        output_files(root)
+          .reject { |file| recorded.key?(file) || excluded?(file, excluded) }
+          .map { |file| "outputs: unrecorded #{file}" }
+      end.sort
+    end
+
+    def self.output_files(root)
+      full_root = File.expand_path(root)
+      return [] unless File.directory?(full_root)
+
+      files([full_root]).map { |file| relative(file) }.uniq
+    end
+
+    def self.expanded(paths)
+      Array(paths).map { |path| File.expand_path(path) }
+    end
+
+    def self.excluded?(relative_path, excluded)
+      full = File.expand_path(relative_path)
+      excluded.any? do |path|
+        full == path || full.start_with?("#{path}#{File::SEPARATOR}")
+      end
+    end
+
     # @param paths [Array<String>] files or directories
+    # @param exclude [Array<String>] expanded paths to skip
     # @return [Hash{String=>String}] relative path → sha256
-    def self.digests(paths)
-      files = paths.flat_map do |path|
-        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
-      end
-      files.select { |file| File.file?(file) }
+    def self.digests(paths, exclude: [])
+      files(paths)
+        .reject { |file| excluded?(file, exclude) }
         .map { |file| relative(file) }
         .uniq.sort
         .to_h { |file| [file, Digest::SHA256.file(file).hexdigest] }
     end
 
+    def self.files(paths)
+      paths.flat_map do |path|
+        File.directory?(path) ? Dir.glob(File.join(path, "**/*")) : [path]
+      end.select { |file| File.file?(file) }
+    end
+
+    # Relative paths of the directory entries among +outputs+.
+    def self.output_roots(outputs, exclude: [])
+      outputs.select { |path| File.directory?(path) }
+        .map { |path| relative(path) }
+        .reject { |path| excluded?(path, exclude) }
+        .uniq.sort
+    end
+
     def self.relative(path)
       Pathname.new(File.expand_path(path))
         .relative_path_from(Pathname.getwd)
@@ -154,6 +231,9 @@ module Suma
       path.to_s
     end
 
-    private_class_method :optional_versions, :sign_with, :executable?
+    private_class_method :optional_versions, :sign_with, :executable?,
+                         :render_gems, :render_gem?, :gem_specs,
+                         :digest_problems, :unrecorded_outputs, :output_files,
+                         :expanded, :excluded?, :files
   end
 end

data/lib/suma/schema_collection.rb

@@ -125,6 +125,7 @@ module Suma
         schema: express,
         output_path: @output_path_docs.join(express.id),
         template: template_class.new(express.id),
+        no_valid: @options[:no_valid],
       )
 
       @docs[express.id] = compiler

data/lib/suma/schema_compiler.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require "digest"
 require "fileutils"
+require "json"
 require "pathname"
 
 module Suma
@@ -12,20 +14,32 @@ module Suma
   # that the caller injects. This split means templates can be tested as pure
   # functions and the compiler can be tested with a real adoc fixture, without
   # either depending on the other.
+  #
+  # Rendering is skipped when the adoc, the schema config, the schema source,
+  # and the rendering toolchain are unchanged since the outputs were written:
+  # the render key is stored beside the outputs.
   class SchemaCompiler
+    RENDER_KEY_FILE = ".suma-render-key"
+
     attr_reader :schema, :id, :output_path, :template
 
-    def initialize(schema:, output_path:, template:)
+    def initialize(schema:, output_path:, template:, no_valid: false)
       @schema = schema
       @id = schema.id
       @output_path = output_path
       @template = template
+      @no_valid = no_valid
     end
 
     def compile
       save_config
-      save_adoc
+      adoc = template.render(config_relative_path)
+      save_adoc(adoc)
+      key = render_key(adoc)
+      return reuse_render if rendered_current?(key)
+
       invoke_metanorma
+      write_render_key(key)
       self
     end
 
@@ -37,6 +51,13 @@ module Suma
       template.extensions
     end
 
+    # Options handed to Metanorma::Compile. Exposed for specs.
+    def compile_options
+      { agree_to_terms: true, install_fonts: false }.tap do |options|
+        options[:novalid] = true if @no_valid
+      end
+    end
+
     private
 
     def filename_adoc(ext = "adoc")
@@ -47,15 +68,20 @@ module Suma
       File.join(output_path, "schema_#{id}.yaml")
     end
 
-    def save_adoc
-      log_relative "Save EXPRESS adoc", filename_adoc
-
-      FileUtils.mkdir_p(File.dirname(filename_adoc))
+    def render_key_path
+      File.join(output_path, RENDER_KEY_FILE)
+    end
 
-      config_relative = Pathname.new(filename_config)
+    def config_relative_path
+      Pathname.new(filename_config)
         .relative_path_from(Pathname.new(File.dirname(filename_adoc)))
+    end
+
+    def save_adoc(content)
+      log_relative "Save EXPRESS adoc", filename_adoc
 
-      File.write(filename_adoc, template.render(config_relative))
+      FileUtils.mkdir_p(File.dirname(filename_adoc))
+      File.write(filename_adoc, content)
     end
 
     def save_config
@@ -68,16 +94,56 @@ module Suma
       config.save_to_path(filename_config)
     end
 
+    # Everything the rendered document depends on. The schema source is an
+    # input even though the adoc references it only by path.
+    def render_key(adoc)
+      Digest::SHA256.hexdigest(
+        [adoc, File.read(filename_config), schema_source_digest,
+         JSON.generate(Provenance.render_toolchain)].join("\0"),
+      )
+    end
+
+    def schema_source_digest
+      path = schema.path.to_s
+      File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing:#{path}"
+    end
+
+    def rendered_current?(key)
+      File.file?(render_key_path) && File.read(render_key_path) == key &&
+        expected_output_paths.all? { |path| File.file?(path) }
+    end
+
+    def expected_output_paths
+      extensions.split(",").map { |ext| filename_adoc(ext.strip) }
+    end
+
+    def write_render_key(key)
+      File.write(render_key_path, key)
+    end
+
+    def reuse_render
+      log_relative "Reusing rendered schema doc (id: #{id})", filename_adoc
+      self
+    end
+
     def invoke_metanorma
       log_relative "Compiling schema (id: #{id})", filename_adoc
-      Metanorma::Compile.new.compile(
-        filename_adoc,
-        agree_to_terms: true,
-        install_fonts: false,
-      )
+      Metanorma::Compile.new.compile(filename_adoc, **compile_options)
+      verify_outputs!
       log_relative "Compiling schema (id: #{id}) ... done!", filename_adoc
     end
 
+    # Metanorma::Compile#compile can return nil for a handled failure instead
+    # of raising. Never cache a render that produced no output.
+    def verify_outputs!
+      missing = expected_output_paths.reject { |path| File.file?(path) }
+      return if missing.empty?
+
+      names = missing.map { |path| File.basename(path) }.join(", ")
+      raise CompilationError,
+            "Metanorma produced no #{names} for schema #{id}"
+    end
+
     def log_relative(prefix, path)
       relative = Pathname.new(path).relative_path_from(Dir.pwd)
       Utils.log "#{prefix}: #{relative}"

data/lib/suma/schema_exporter.rb

@@ -47,6 +47,18 @@ module Suma
     end
 
     def export_to_directory(schemas)
+      run_pool_export(schemas)
+    rescue SystemStackError
+      # One pathological schema (a very deep AST) can overflow the
+      # worker threads' ~1 MB stack. The main thread has the full OS
+      # stack and every completed export is cached, so the retry only
+      # pays for the schemas that did not finish.
+      Utils.log "[export] worker thread stack exhausted; " \
+                "retrying the remaining schemas on the main thread"
+      schemas.each { |schema| export_single_schema(schema) }
+    end
+
+    def run_pool_export(schemas)
       WorkerPool.new(size: options[:jobs]).map(schemas) do |schema|
         export_single_schema(schema)
       end

data/lib/suma/schema_index.rb

@@ -40,11 +40,11 @@ module Suma
 
     def element_collections(schema)
       [
-        schema.entities,
-        schema.types,
-        schema.constants,
-        schema.functions,
-        schema.rules,
+        Array(schema.entities),
+        Array(schema.types),
+        Array(schema.constants),
+        Array(schema.functions),
+        Array(schema.rules),
         schema.procedures,
         schema.subtype_constraints,
       ]

data/lib/suma/schema_manifest_generator.rb

@@ -48,11 +48,18 @@ exclude_paths: nil)
       YAML.safe_load(File.read(file_path, encoding: "UTF-8"), aliases: true)
     end
 
+    # Collection manifest files referenced by the site manifest. Both
+    # manifest shapes are handled: legacy `docref` entries and the
+    # nested `documents` entries (metanorma 2.x collection manifests).
+    # Only `file` references count — `fileref` entries are attachments
+    # (documents, schema exports), not collection manifests.
     def load_manifest_files(collection_files)
-      collection_files.map do |c|
+      collection_files.flat_map do |c|
         collection_data = load_yaml(c)
-        collection_data["manifest"]["docref"].map { |docref| docref["file"] }
-      end.flatten
+        manifest = collection_data["manifest"] || {}
+        entries = Array(manifest["docref"]) + Array(manifest["documents"])
+        entries.filter_map { |entry| entry["file"] }
+      end
     end
 
     def load_project_schemas(manifest_files)

data/lib/suma/schema_profile.rb

@@ -126,9 +126,21 @@ module Suma
       category_ids: %w[other],
     )
 
+    # buildingSMART IFC: one standalone schema per edition (IFC4, IFC4X3,
+    # ...), entities prefixed Ifc, no ARM/MIM-style decomposition.
+    IFC = new(
+      id: "ifc",
+      rules: [],
+      suffix_labels: {},
+      acronyms: %w[bim ifc idf 2d 3d 4d 5d],
+      acronym_patterns: [/\Aifc\d+(x\d+)*\z/],
+      category_ids: %w[other],
+    )
+
     REGISTRY = {
       "iso10303" => ISO_10303,
       "generic" => GENERIC,
+      "ifc" => IFC,
     }.freeze
   end
 end