suma 0.5.9 → 0.5.11

data/lib/suma/schema_template/anchors.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Suma
+  module SchemaTemplate
+    # The Liquid anchor block shared by the schema document templates: one
+    # [[id]] for every element an Expressir cross-reference may target, so
+    # every xref the hyperlinked formatter emits resolves in the rendered
+    # document. Expressir links schema-qualified element ids
+    # (`schema.entity`) but bare `entity.attribute` pairs, hence the two
+    # bookmark forms.
+    module Anchors
+      # Xref targets are always qualified by the schema's *declared* id, so
+      # anchors are too — independent of the manifest key's casing.
+      SCHEMA_REF = "{{ schema.id }}"
+
+      # The raw anchor block contains Liquid comments and newlines that are
+      # illegal on the section header line; strip them before interpolating.
+      def rendered_anchors
+        anchor_block
+          .gsub(%r{//[^\r\n]+}, "")
+          .gsub(/[\n\r]+/, "")
+          .gsub(/^[\n\r]/, "")
+      end
+
+      private
+
+      def anchor_block
+        <<~HEREDOC
+          // _fund_cons.liquid
+          [[#{schema_id}_funds]]
+
+          #{constants_and_types_anchors}
+
+          #{entity_anchors}
+
+          #{declaration_anchors}
+        HEREDOC
+      end
+
+      def constants_and_types_anchors
+        <<~HEREDOC
+          // _constants.liquid
+          {% if schema.constants.size > 0 %}
+          #{bookmark('constants')}
+          {% for thing in schema.constants %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _types.liquid
+          {% if schema.types.size > 0 %}
+          #{bookmark('types')}
+          // _type.liquid
+          {% for thing in schema.types %}
+          #{bookmark('{{thing.id}}')}
+          {% if thing.items.size > 0 %}
+          // _type_items.liquid
+          #{bookmark('{{thing.id}}.items')}
+          {% for item in thing.items %}
+          #{bookmark('{{thing.id}}.items.{{item.id}}')}
+          {% endfor %}
+          {% endif %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def entity_anchors
+        <<~HEREDOC
+          // _entities.liquid
+          {% if schema.entities.size > 0 %}
+          #{bookmark('entities')}
+          // _entity.liquid
+          {% for thing in schema.entities %}
+          #{bookmark('{{thing.id}}')}
+          {% for attr in thing.attributes %}
+          #{bookmark_local('{{thing.id}}.{{attr.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      def declaration_anchors
+        <<~HEREDOC
+          // _subtype_constraints.liquid
+          {% if schema.subtype_constraints.size > 0 %}
+          #{bookmark('subtype_constraints')}
+          // _subtype_constraint.liquid
+          {% for thing in schema.subtype_constraints %}
+          #{bookmark('{{thing.id}}')}
+          {% endfor %}
+          {% endif %}
+
+          // _functions.liquid
+          {% if schema.functions.size > 0 %}
+          #{bookmark('functions')}
+          // _function.liquid
+          {% for thing in schema.functions %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _procedures.liquid
+          {% if schema.procedures.size > 0 %}
+          #{bookmark('procedures')}
+          // _procedure.liquid
+          {% for thing in schema.procedures %}
+          #{callable_anchors}
+          {% endfor %}
+          {% endif %}
+
+          // _rules.liquid
+          {% if schema.rules.size > 0 %}
+          #{bookmark('rules')}
+          // _rule.liquid
+          {% for thing in schema.rules %}
+          #{bookmark('{{thing.id}}')}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+          {% endfor %}
+          {% endif %}
+        HEREDOC
+      end
+
+      # Functions and procedures carry parameters and local variables, and
+      # Expressir links references to them as bare `owner.name` pairs.
+      def callable_anchors
+        <<~HEREDOC
+          #{bookmark('{{thing.id}}')}
+          {% for param in thing.parameters %}
+          #{bookmark_local('{{thing.id}}.{{param.id}}')}
+          {% endfor %}
+          {% for var in thing.variables %}
+          #{bookmark_local('{{thing.id}}.{{var.id}}')}
+          {% endfor %}
+        HEREDOC
+      end
+
+      def bookmark(anchor)
+        "[[#{mangled(SCHEMA_REF)}.#{mangled(anchor)}]]"
+      end
+
+      def bookmark_local(anchor)
+        "[[#{mangled(anchor)}]]"
+      end
+
+      def mangled(anchor)
+        anchor.gsub("}}", ' | replace: "\", "-", "-}}')
+      end
+    end
+  end
+end

data/lib/suma/schema_template/document.rb

@@ -4,10 +4,11 @@ module Suma
   module SchemaTemplate
     # Emits an AsciiDoc body with cross-reference anchors for every
     # schema element so other documents can deep-link into the compiled
-    # HTML. Only XML is produced — the anchors only resolve against the
+    # output. Only XML is produced — the anchors only resolve against the
     # XML output, not the HTML rendering.
     class Document
       EXTENSIONS = "xml"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -44,98 +45,6 @@ module Suma
 
         ADOC
       end
-
-      private
-
-      # The raw anchor block contains Liquid comments and newlines that
-      # are illegal on the section header line; strip them before
-      # interpolating.
-      def rendered_anchors
-        schema_anchors.gsub(%r{//[^\r\n]+}, "").gsub(/[\n\r]+/, "").gsub(
-          /^[\n\r]/, ""
-        )
-      end
-
-      def schema_anchors
-        <<~HEREDOC
-          // _fund_cons.liquid
-          [[#{schema_id}_funds]]
-
-          // _constants.liquid
-          {% if schema.constants.size > 0 %}
-          #{bookmark('constants')}
-          {% for thing in schema.constants %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _types.liquid
-          {% if schema.types.size > 0 %}
-          #{bookmark('types')}
-          // _type.liquid
-          {% for thing in schema.types %}
-          #{bookmark('{{thing.id}}')}
-          {% if thing.items.size > 0 %}
-          // _type_items.liquid
-          #{bookmark('{{thing.id}}.items')}
-          {% for item in thing.items %}
-          #{bookmark('{{thing.id}}.items.{{item.id}}')}
-          {% endfor %}
-          {% endif %}
-          {% endfor %}
-          {% endif %}
-
-          // _entities.liquid
-          {% if schema.entities.size > 0 %}
-          #{bookmark('entities')}
-          {% for thing in schema.entities %}
-          // _entity.liquid
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _subtype_constraints.liquid
-          {% if schema.subtype_constraints.size > 0 %}
-          #{bookmark('subtype_constraints')}
-          // _subtype_constraint.liquid
-          {% for thing in schema.subtype_constraints %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _functions.liquid
-          {% if schema.functions.size > 0 %}
-          #{bookmark('functions')}
-          // _function.liquid
-          {% for thing in schema.functions %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _procedures.liquid
-          {% if schema.procedures.size > 0 %}
-          #{bookmark('procedures')}
-          // _procedure.liquid
-          {% for thing in schema.procedures %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-
-          // _rules.liquid
-          {% if schema.rules.size > 0 %}
-          #{bookmark('rules')}
-          // _rule.liquid
-          {% for thing in schema.rules %}
-          #{bookmark('{{thing.id}}')}
-          {% endfor %}
-          {% endif %}
-        HEREDOC
-      end
-
-      def bookmark(anchor)
-        mangled = anchor.gsub("}}", ' | replace: "\", "-", "-}}')
-        "[[#{schema_id}.#{mangled}]]"
-      end
     end
   end
 end

data/lib/suma/schema_template/plain.rb

@@ -2,10 +2,13 @@
 
 module Suma
   module SchemaTemplate
-    # Emits a plain AsciiDoc body for a single EXPRESS schema, producing
-    # both HTML and XML outputs via Metanorma.
+    # Emits an anchored, hyperlinked AsciiDoc body for a single EXPRESS
+    # schema, producing both HTML and XML outputs via Metanorma. Every
+    # element cross-reference the hyperlinked source emits resolves inside
+    # the rendered document through the shared anchor block.
     class Plain
       EXTENSIONS = "xml,html"
+      include Anchors
 
       attr_reader :schema_id
 
@@ -29,12 +32,13 @@ module Suma
           ----
           {% for schema in context.schemas %}
 
-          [%unnumbered]
-          == #{schema_id}
+          [[#{schema_id}]]
+          [%unnumbered,type=express]
+          == #{schema_id} #{rendered_anchors}
 
-          [source%unnumbered]
+          [source%unnumbered,subs="+macros"]
           --
-          {{ schema.formatted }}
+          {{ schema.formatted_hyperlinked_adoc }}
           --
           {% endfor %}
           ----

data/lib/suma/schema_template.rb

@@ -13,6 +13,7 @@ module Suma
   #
   # Composition with the compiler lives in SchemaCompiler.
   module SchemaTemplate
+    autoload :Anchors, "suma/schema_template/anchors"
     autoload :Plain, "suma/schema_template/plain"
     autoload :Document, "suma/schema_template/document"
   end

data/lib/suma/site_build.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "fileutils"
+require "pathname"
+
+module Suma
+  # Freshness decision for the rendered collection site.
+  #
+  # The collection render is expensive: Metanorma compiles every member
+  # document. Its output is a pure function of the emitted collection manifest,
+  # the source tree it references, and the rendering toolchain, so this class
+  # keys those inputs, remembers the key and the produced files in the output
+  # directory, and lets the processor skip the render when nothing changed.
+  class SiteBuild
+    STATE_FILE = ".suma-site-build.json"
+
+    # Emitted manifests that the build itself regenerates deterministically
+    # from the sources above; whether they exist yet must not change the key.
+    GENERATED_FILES = %w[collection-output.yaml schemas.yml].freeze
+
+    def initialize(root:, output_directory:, inputs: [])
+      @root = File.expand_path(root)
+      @output_directory = File.expand_path(output_directory)
+      @extra_inputs = inputs.map { |path| File.expand_path(path) }
+    end
+
+    # @return [Array<String>] source files determining the rendered site
+    def source_files
+      @source_files ||= begin
+        files = @extra_inputs.select { |file| File.file?(file) }
+        (files + tree_files).uniq.sort
+      end
+    end
+
+    # @return [String] digest of every source file and the render toolchain
+    def key
+      @key ||= begin
+        digest = Digest::SHA256.new
+        source_files.each do |file|
+          digest << relative(file) << "\0"
+          digest << Digest::SHA256.file(file).hexdigest << "\0"
+        end
+        digest << JSON.generate(Provenance.render_toolchain)
+        digest.hexdigest
+      end
+    end
+
+    # True when the previous render used the same inputs and every file it
+    # produced is still present.
+    def current?
+      state = read_state
+      return false unless state && state["key"] == key
+
+      outputs = Array(state["outputs"])
+      !outputs.empty? &&
+        outputs.all? { |file| File.file?(File.join(@output_directory, file)) }
+    end
+
+    # Persist the key and the produced files for the next build.
+    def record
+      write_state("key" => key, "outputs" => output_files)
+      self
+    end
+
+    def state_path
+      File.join(@output_directory, STATE_FILE)
+    end
+
+    private
+
+    def tree_files
+      excluded = excluded_paths
+      Dir.glob(File.join(@root, "**", "*")).select do |file|
+        File.file?(file) &&
+          excluded.none? { |path| under?(file, path) } &&
+          !generated?(relative(file).tr("\\", "/"))
+      end
+    end
+
+    # Files the collection render writes next to its sources. They are
+    # outputs of the previous build, never inputs, so including them would
+    # change the key on every run.
+    def generated?(relative)
+      base = File.basename(relative)
+      GENERATED_FILES.include?(relative) ||
+        relative.start_with?("relaton/") ||
+        base.start_with?("tmp_document") ||
+        base.end_with?("_index.html", ".err.html", ".log.txt") ||
+        document_artifact?(relative, base)
+    end
+
+    def document_artifact?(relative, base)
+      return false unless relative.start_with?("documents/")
+
+      base == "cover.html" || base == "document.xml" ||
+        (base.start_with?("document.") && base.include?(".xml"))
+    end
+
+    # Generated trees are derived from the sources above and never feed the
+    # site render as inputs of their own. Bundler's install directory is a
+    # dependency artifact: CI conventionally places it at vendor/bundle
+    # inside the project, where it must never count as site source.
+    def excluded_paths
+      fixed = %w[schema_docs plain_schemas .git .bundle vendor/bundle]
+        .map { |dir| File.join(@root, dir) }
+      env = [ENV.fetch("SUMA_SCHEMA_CACHE_DIR", nil),
+             ENV.fetch("BUNDLE_PATH", nil)]
+        .compact
+        .map { |dir| File.expand_path(dir, @root) }
+      [@output_directory] + fixed + env
+    end
+
+    def under?(file, path)
+      file == path || file.start_with?("#{path}#{File::SEPARATOR}")
+    end
+
+    def read_state
+      return unless File.file?(state_path)
+
+      JSON.parse(File.read(state_path))
+    rescue JSON::ParserError
+      nil
+    end
+
+    def write_state(state)
+      FileUtils.mkdir_p(@output_directory)
+      temp = "#{state_path}.#{Process.pid}.tmp"
+      File.write(temp, "#{JSON.pretty_generate(state)}\n")
+      File.rename(temp, state_path)
+    ensure
+      File.delete(temp) if temp && File.exist?(temp)
+    end
+
+    def output_files
+      Dir.glob(File.join(@output_directory, "**", "*"))
+        .select do |file|
+          File.file?(file) && File.basename(file) != STATE_FILE
+        end
+        .map { |file| relative_to(file, @output_directory) }
+        .sort
+    end
+
+    def relative(path)
+      relative_to(path, @root)
+    end
+
+    def relative_to(path, base)
+      Pathname.new(path).relative_path_from(Pathname.new(base)).to_s
+    end
+  end
+end

data/lib/suma/staged_collection_builder.rb

@@ -16,10 +16,12 @@ module Suma
   #      stub-bearing Semantic XML + anchors to a durable content-addressed store
   #      (the metanorma engine's +preserve_unresolved:+ / +artifact_store_dir:+
   #      render options, metanorma/metanorma#576);
-  #   2. runs those per-member processes SEQUENTIALLY (never in parallel --
-  #      parallelism reintroduces the cross-linking / sectionsplit races that are
-  #      deliberately avoided); each process exits and the OS reclaims its memory
-  #      before the next, so peak memory is bounded by a single member; then
+  #   2. runs those per-member processes SEQUENTIALLY by default; with
+  #      +workers+ > 1, batches of members are rendered by that many child
+  #      processes concurrently (safe: members write disjoint store entries,
+  #      and sectionsplit happens only in the sequential reinflate pass); each
+  #      process exit hands its memory back to the OS, so peak memory stays
+  #      bounded by workers x one member; then
   #   3. REINFLATES: a final pass assembles a manifest of the stored stubs and
   #      resolves them into real cross-document links (+reinflate:+), producing
   #      the same output an all-present build would.
@@ -34,31 +36,159 @@ module Suma
   class StagedCollectionBuilder
     STORE_DIRNAME = ".metanorma-collection-cache"
 
+    # Members rendered per child process: one Metanorma boot amortised over
+    # a batch of small members keeps boot cost under a few percent.
+    DEFAULT_BATCH_SIZE = 25
+
     # +collection_config_path+ is the emitted collection manifest (the same
     # +collection-output.yaml+ the normal build renders). Member filerefs are
     # relative to its directory, so all per-member manifests are written there.
+    # +workers+ > 1 runs that many renderer processes concurrently over batches
+    # of +batch_size+ members; members write disjoint, identifier-keyed store
+    # entries, so children cannot race. (a data-declaring initializer)
     def initialize(collection_config_path:, output_directory:, coverpage: nil,
-                   store_dir: nil, formats: %i[xml html])
+                   store_dir: nil, formats: %i[xml html], no_valid: false,
+                   workers: 1, batch_size: DEFAULT_BATCH_SIZE) # rubocop:disable Metrics/ParameterLists
       @config_path = collection_config_path
       @config_dir = File.dirname(File.expand_path(collection_config_path))
       @output_directory = output_directory
       @coverpage = coverpage
       @store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
       @formats = formats
+      @no_valid = no_valid
+      @workers = [workers.to_i, 1].max
+      @batch_size = [batch_size.to_i, 1].max
     end
 
     def build
       FileUtils.mkdir_p(@output_directory)
       FileUtils.mkdir_p(@store_dir)
       members = collection_members
-      Utils.log "[staged] #{members.size} members; one process per member, " \
-                "sequential; store: #{@store_dir}"
-      members.each_with_index { |member, i| stage_member(member, i) }
+      Utils.log "[staged] #{members.size} members; workers: #{@workers}; " \
+                "batch size: #{@batch_size}; store: #{@store_dir}"
+      if @workers == 1
+        members.each_with_index { |member, i| stage_member(member, i) }
+      else
+        stage_parallel(members)
+      end
       reinflate(members)
     end
 
     private
 
+    # Batch the members and run +@workers+ renderer child processes
+    # concurrently. Each batch file carries fully-rendered single-member
+    # manifests; children claim batches through O_EXCL markers, so a slow
+    # worker never starves. One Metanorma boot amortises over
+    # +@batch_size+ members (the boot dominates small members), and each
+    # child exit hands its memory back to the OS. Members write disjoint,
+    # identifier-keyed store entries, so concurrent children cannot race;
+    # sectionsplit only happens at reinflate (a single sequential pass).
+    def stage_parallel(members)
+      paths, count = write_batch_files(members)
+      run_batch_workers(count)
+    ensure
+      remove_batch_files(paths, count)
+    end
+
+    def write_batch_files(members)
+      batches = batch_manifests(members)
+      paths = write_batch_bins(batches)
+      Utils.log "[staged] #{batches.size} batches over #{@workers} workers"
+      [paths, batches.size]
+    end
+
+    def batch_manifests(members)
+      members.each_slice(@batch_size).map do |slice|
+        slice.map do |member|
+          base_manifest.merge("manifest" => collection_manifest([member]))
+        end
+      end
+    end
+
+    def write_batch_bins(batches)
+      batches.map.with_index do |batch, i|
+        path = File.join(@config_dir, "._suma_batch_#{i}.bin")
+        File.binwrite(path, Marshal.dump(batch))
+        path
+      end
+    end
+
+    def run_batch_workers(count)
+      children = Array.new([@workers, count].min) do
+        Process.spawn(RbConfig.ruby, "-e", child_script, @config_dir,
+                      count.to_s, @store_dir, @output_directory,
+                      @no_valid ? "1" : "0", chdir: @config_dir)
+      end
+      failed = children
+        .filter_map { |pid| Process.waitpid2(pid) }
+        .map(&:last).reject(&:success?).size
+      return if failed.zero?
+
+      raise "[staged] #{failed} of #{@workers} batch workers failed; " \
+            "see the worker output above"
+    end
+
+    def remove_batch_files(paths, count)
+      paths&.each { |path| FileUtils.rm_f(path) }
+      (0...(count || 0)).each do |i|
+        FileUtils.rm_f(File.join(@config_dir, "._suma_batch_#{i}.claim"))
+      end
+    end
+
+    # Child renderer: claims and renders batches until none remain. The
+    # Metanorma boot is paid once per child, and GC runs between members to
+    # bound the heap. Any render failure exits nonzero and fails the build.
+    def child_script
+      <<~RUBY
+        require "yaml"
+        require "fileutils"
+        require "metanorma"
+        config_dir, batch_count, store_dir, out_dir, no_valid = ARGV
+        count = Integer(batch_count)
+        claimed = 0
+        loop do
+          idx = (0...count).find do |i|
+            begin
+              File.open(File.join(config_dir, "._suma_batch_\#{i}.claim"),
+                        File::RDWR | File::CREAT | File::EXCL) {}
+              true
+            rescue Errno::EEXIST
+              false
+            end
+          end
+          break unless idx
+
+          manifests = Marshal.load(File.binread(File.join(
+            config_dir, "._suma_batch_\#{idx}.bin")))
+          manifests.each_with_index do |manifest, j|
+            one_path = File.join(config_dir,
+                                 "._suma_staged_b\#{idx}_\#{j}.yml")
+            out = File.join(out_dir, "._staged_out_b\#{idx}_\#{j}")
+            t0 = Time.now
+            begin
+              File.write(one_path, YAML.dump(manifest))
+              compile = { install_fonts: false }
+              compile[:novalid] = true if no_valid == "1"
+              Metanorma::Collection.parse(one_path).render(
+                format: [:xml], output_folder: out,
+                preserve_unresolved: true, artifact_store_dir: store_dir,
+                compile: compile,
+              )
+              warn format("[staged worker %d] batch %d/%d member %d OK (%ds)",
+                          Process.pid, idx + 1, count, j, Time.now - t0)
+            ensure
+              File.unlink(one_path) rescue nil
+              FileUtils.rm_rf(out) rescue nil
+              GC.start
+            end
+          end
+          claimed += 1
+        end
+        exit 1 if claimed.zero? && count.positive?
+      RUBY
+    end
+
     def manifest
       @manifest ||= YAML.load_file(@config_path)
     end
@@ -97,7 +227,7 @@ module Suma
         manifest_path: one_path,
         opts: { format: [:xml], output_folder: out,
                 preserve_unresolved: true, artifact_store_dir: @store_dir,
-                compile: { install_fonts: false } },
+                compile: compile_flags },
       )
     ensure
       FileUtils.rm_f(one_path)
@@ -118,10 +248,16 @@ module Suma
         manifest_path: reinf_path,
         opts: { format: @formats, output_folder: @output_directory,
                 reinflate: true, coverpage: @coverpage,
-                compile: { install_fonts: false } }.compact,
+                compile: compile_flags }.compact,
       )
     end
 
+    def compile_flags
+      { install_fonts: false }.tap do |flags|
+        flags[:novalid] = true if @no_valid
+      end
+    end
+
     # Copy each member's stored stub into +dir+ and return docref entries pointing
     # at the copies, keyed by real docid (as the store keys them).
     def stored_docrefs(members, dir)