suma 0.4.0 → 0.5.1

data/lib/suma/staged_collection_builder.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require "metanorma"
+require "yaml"
+require "fileutils"
+
+module Suma
+  # Memory-bounded, resumable staged collection build (metanorma/suma#94).
+  #
+  # A normal collection build renders every member document in one OS process, so
+  # peak memory accumulates across the whole collection and a large collection
+  # (e.g. the STEP SRL) OOMs. This builder instead:
+  #
+  #   1. compiles each member in its OWN fresh OS process, preserving that
+  #      member's unresolved cross-document references as stubs and writing the
+  #      stub-bearing Semantic XML + anchors to a durable content-addressed store
+  #      (the metanorma engine's +preserve_unresolved:+ / +artifact_store_dir:+
+  #      render options, metanorma/metanorma#576);
+  #   2. runs those per-member processes SEQUENTIALLY (never in parallel --
+  #      parallelism reintroduces the cross-linking / sectionsplit races that are
+  #      deliberately avoided); each process exits and the OS reclaims its memory
+  #      before the next, so peak memory is bounded by a single member; then
+  #   3. REINFLATES: a final pass assembles a manifest of the stored stubs and
+  #      resolves them into real cross-document links (+reinflate:+), producing
+  #      the same output an all-present build would.
+  #
+  # The empirical case (metanorma/metanorma#576): on the real SRL a single-process
+  # build peaks ~4.1 GB and OOMs; each member compiled in isolation peaks ~1 GB,
+  # so process isolation is a ~4x peak reduction that converts the OOM into a
+  # bounded footprint.
+  #
+  # This is opt-in; +Suma::Processor+ uses the normal single-process build unless
+  # staging is requested.
+  class StagedCollectionBuilder
+    STORE_DIRNAME = ".metanorma-collection-cache"
+
+    # +collection_config_path+ is the emitted collection manifest (the same
+    # +collection-output.yaml+ the normal build renders). Member filerefs are
+    # relative to its directory, so all per-member manifests are written there.
+    def initialize(collection_config_path:, output_directory:, coverpage: nil,
+                   store_dir: nil, formats: %i[xml html])
+      @config_path = collection_config_path
+      @config_dir = File.dirname(File.expand_path(collection_config_path))
+      @output_directory = output_directory
+      @coverpage = coverpage
+      @store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
+      @formats = formats
+    end
+
+    def build
+      FileUtils.mkdir_p(@output_directory)
+      FileUtils.mkdir_p(@store_dir)
+      members = collection_members
+      Utils.log "[staged] #{members.size} members; one process per member, " \
+                "sequential; store: #{@store_dir}"
+      members.each_with_index { |member, i| stage_member(member, i) }
+      reinflate(members)
+    end
+
+    private
+
+    def manifest
+      @manifest ||= YAML.load_file(@config_path)
+    end
+
+    # Every document member of the manifest tree, in order: any node carrying a
+    # file reference (+fileref+ or legacy +file+) that is not an attachment.
+    def collection_members
+      members = []
+      walk_members(manifest["manifest"]) { |member| members << member }
+      members
+    end
+
+    def walk_members(node, &block)
+      case node
+      when Array then node.each { |child| walk_members(child, &block) }
+      when Hash then walk_member_hash(node, &block)
+      end
+    end
+
+    def walk_member_hash(node, &block)
+      ref = node["fileref"] || node["file"]
+      ref && !truthy?(node["attachment"]) and
+        yield({ "fileref" => ref, "identifier" => node["identifier"],
+                "sectionsplit" => node["sectionsplit"] }.compact)
+      node.each_value { |value| walk_members(value, &block) }
+    end
+
+    # Stage one member in its own process: a single-member manifest rendered with
+    # preserve + store. Written beside the collection config so relative filerefs
+    # resolve; removed afterwards.
+    def stage_member(member, idx)
+      one_path = File.join(@config_dir, "._suma_staged_#{idx}.yml")
+      File.write(one_path, single_member_manifest(member).to_yaml)
+      out = File.join(@output_directory, "._staged_out_#{idx}")
+      run_render(
+        manifest_path: one_path,
+        opts: { format: [:xml], output_folder: out,
+                preserve_unresolved: true, artifact_store_dir: @store_dir,
+                compile: { install_fonts: false } },
+      )
+    ensure
+      FileUtils.rm_f(one_path)
+      FileUtils.rm_rf(File.join(@output_directory, "._staged_out_#{idx}"))
+    end
+
+    # Assemble a manifest of the stored stubs and reinflate them into the site.
+    def reinflate(members)
+      reinf_dir = File.join(@output_directory, "_staged_reinflate")
+      FileUtils.mkdir_p(reinf_dir)
+      docrefs = stored_docrefs(members, reinf_dir)
+      reinf_path = File.join(reinf_dir, "reinflate.yml")
+      File.write(reinf_path,
+                 base_manifest.merge("manifest" => collection_manifest(docrefs)).to_yaml)
+      Utils.log "[staged] reinflating #{docrefs.size} stored members -> " \
+                "#{@output_directory}"
+      run_render(
+        manifest_path: reinf_path,
+        opts: { format: @formats, output_folder: @output_directory,
+                reinflate: true, coverpage: @coverpage,
+                compile: { install_fonts: false } }.compact,
+      )
+    end
+
+    # Copy each member's stored stub into +dir+ and return docref entries pointing
+    # at the copies, keyed by real docid (as the store keys them).
+    def stored_docrefs(members, dir)
+      members.map.with_index do |member, i|
+        stored = stored_semantic(member["identifier"]) or
+          raise "[staged] no stored artefact for #{member['identifier'].inspect}"
+        name = "member_#{i}.xml"
+        FileUtils.cp(stored, File.join(dir, name))
+        dr = { "fileref" => name, "identifier" => member["identifier"] }
+        dr["sectionsplit"] = true if truthy?(member["sectionsplit"])
+        dr
+      end
+    end
+
+    # A minimal single-member manifest reusing the collection's directives and
+    # bibdata, so the isolated render carries the right collection identity.
+    def single_member_manifest(member)
+      base_manifest.merge("manifest" => collection_manifest([member]))
+    end
+
+    def collection_manifest(docrefs)
+      { "level" => "collection", "title" => "Collection",
+        "manifest" => [{ "level" => "subcollection", "title" => "Members",
+                         "docref" => docrefs }] }
+    end
+
+    def base_manifest
+      { "directives" => manifest["directives"] || ["documents-inline"],
+        "bibdata" => manifest["bibdata"] }.compact
+    end
+
+    def stored_semantic(identifier)
+      Dir[File.join(@store_dir, "#{slug(identifier)}.*.semantic.xml")]
+        .max_by { |f| File.mtime(f) }
+    end
+
+    # Mirrors the store's filename slug (metanorma ArtifactStore#slug).
+    def slug(identifier)
+      identifier.to_s.gsub(/[^A-Za-z0-9._-]+/, "-")
+        .gsub(/-{2,}/, "-").gsub(/\A-|-\z/, "")
+    end
+
+    # Render a manifest in a FRESH child process so its memory is reclaimed on
+    # exit. Under +bundle exec+, the child inherits the bundle. Raises on failure.
+    def run_render(manifest_path:, opts:)
+      script = <<~RUBY
+        require "metanorma"
+        opts = Marshal.load(STDIN.read)
+        Metanorma::Collection.parse(#{manifest_path.inspect}).render(opts)
+      RUBY
+      IO.popen([RbConfig.ruby, "-e", script], "wb") do |io|
+        io.write(Marshal.dump(opts))
+        io.close_write
+      end
+      return if $?.success?
+
+      raise "[staged] render subprocess failed for #{manifest_path}"
+    end
+
+    def truthy?(val)
+      [true, "true"].include?(val)
+    end
+  end
+end

data/lib/suma/term_extractor.rb

@@ -6,15 +6,6 @@ require "glossarist"
 
 module Suma
   class TermExtractor
-    REDUNDANT_NOTE_REGEX =
-      %r{
-        ^An?                   # Starts with "A" or "An"
-        \s.*?\sis\sa\stype\sof # Text followed by "is a type of"
-        (\sa|\san)?            # Optional " a" or " an"
-        \s\{\{[^\}]*\}\}       # Text in double curly braces
-        \s*?\.?$               # Optional whitespace and period at the end
-      }x
-
     def initialize(schema_manifest_file, output_path, urn:,
                    language_code: "eng")
       @schema_manifest_file = File.expand_path(schema_manifest_file)
@@ -140,7 +131,11 @@ module Suma
         data.domain = schema_domain
         data.sources = [source_ref] if source_ref
 
-        notes = get_entity_notes(entity, schema_domain, data.definition)
+        notes = NoteProcessor.call(
+          entity.remarks,
+          definitions: data.definition,
+          xref_to_mention: method(:xref_to_mention),
+        )
         data.notes = notes if notes && !notes.empty?
         data.examples = []
       end
@@ -166,6 +161,10 @@ module Suma
       "{{#{urn},#{display}}}"
     end
 
+    def xref_to_mention(full_ref, display)
+      urn_mention(express_entity_urn(full_ref), display)
+    end
+
     def get_section_ref(schema)
       return nil unless @urn
 
@@ -219,183 +218,6 @@ module Suma
       [Glossarist::V3::DetailedDefinition.new(content: definition)]
     end
 
-    def get_entity_notes(entity, schema_domain, definitions)
-      notes = []
-
-      if entity.remarks && !entity.remarks.empty?
-        trimmed_def = trim_definition(entity.remarks)
-        if trimmed_def && !trimmed_def.empty?
-          notes << Glossarist::V3::DetailedDefinition.new(
-            content: convert_express_xref(trimmed_def, schema_domain),
-          )
-        end
-      end
-
-      notes = only_keep_first_sentence(notes)
-      notes = remove_see_content(notes)
-      notes = remove_redundant_note(notes)
-      notes = remove_invalid_references(notes)
-      compare_with_definitions(notes, definitions)
-    end
-
-    def only_keep_first_sentence(notes)
-      notes.each do |note|
-        if note&.content && should_preserve_complete_structure?(note.content)
-          next
-        end
-
-        if note&.content
-          new_content = note.content
-            .split(".\n").first.strip
-            .split(". ").first.strip
-          note.content = new_content.end_with?(".") ? new_content : "#{new_content}."
-        end
-      end
-    end
-
-    def should_preserve_complete_structure?(content)
-      return false if content.nil? || content.empty?
-
-      lines = content.split("\n")
-      first_paragraph = lines.first&.strip
-
-      if first_paragraph&.end_with?(":") && lines.length > 1
-        if first_paragraph.count(".").positive?
-          return false
-        end
-
-        remaining_content = lines[1..].join("\n")
-        return starts_with_list?(remaining_content.strip)
-      end
-
-      false
-    end
-
-    def compare_with_definitions(notes, definitions)
-      if notes&.first&.content == definitions&.first&.content
-        return []
-      end
-
-      notes
-    end
-
-    def remove_invalid_references(notes)
-      notes.reject do |note|
-        note.content.include?("image::") ||
-          note.content.match?(/<<(.*?){1,999}>>/)
-      end
-    end
-
-    def remove_redundant_note(notes)
-      notes.reject do |note|
-        note.content.match?(REDUNDANT_NOTE_REGEX) &&
-          !note.content.include?("\n")
-      end
-    end
-
-    def remove_see_content(notes)
-      notes.each do |note|
-        note.content = note.content.gsub(/\s+\(see(.*?){1,999}\)/, "")
-      end
-    end
-
-    def starts_with_list?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\s*[*\-+]\s+/) || content.match?(/^\s*\d+\.\s+/)
-    end
-
-    def trim_definition(definition)
-      return nil if definition.nil? || definition.empty?
-
-      definition_str = definition.is_a?(Array) ? definition.join("\n\n") : definition.to_s
-
-      return nil if definition_str.empty?
-
-      paragraphs = definition_str.split("\n\n")
-      first_paragraph = paragraphs.first
-
-      combined = if paragraphs.length == 1
-                   apply_first_sentence_logic(first_paragraph)
-                 elsif first_paragraph.end_with?(":") && paragraphs.length > 1 && starts_with_list?(paragraphs[1])
-                   complete_list = extract_complete_list(paragraphs, 1)
-                   "#{first_paragraph}\n\n#{complete_list}"
-                 else
-                   apply_first_sentence_logic(first_paragraph)
-                 end
-
-      combined = "#{combined}\n"
-      combined.gsub!(/\n\/\/.*?\n/, "\n")
-      combined.strip!
-
-      express_reference_to_mention(combined)
-    end
-
-    def apply_first_sentence_logic(paragraph)
-      new_content = paragraph
-        .split(".\n").first.strip
-        .split(". ").first.strip
-
-      new_content.end_with?(".") ? new_content : "#{new_content}."
-    end
-
-    def extract_complete_list(paragraphs, start_index)
-      return paragraphs[start_index] if start_index >= paragraphs.length
-
-      combined = paragraphs[start_index].dup
-      current_index = start_index + 1
-      in_continuation_block = combined.include?("--") && !combined.match?(/--.*--/m)
-
-      while current_index < paragraphs.length
-        next_para = paragraphs[current_index]
-
-        if next_para.match?(/^--\s*$/) || next_para.end_with?("--")
-          in_continuation_block = !in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if in_continuation_block
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          next
-        end
-
-        if starts_with_list?(next_para) || is_list_continuation?(next_para)
-          combined += "\n\n#{next_para}"
-          current_index += 1
-          in_continuation_block = true if next_para.include?("--") && !next_para.match?(/--.*--/m)
-        else
-          break
-        end
-      end
-
-      combined
-    end
-
-    def is_list_continuation?(content)
-      return false if content.nil? || content.empty?
-
-      content.match?(/^\+\s*$/) ||
-        content.match?(/^--\s*$/) ||
-        content.match?(/^\s{2,}/) ||
-        content.start_with?("which", "where", "that")
-    end
-
-    def express_reference_to_mention(description)
-      description
-        .gsub(/<<express:([\w.]+)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          entity_id = full_ref.split(".").last
-          urn_mention(express_entity_urn(full_ref), entity_id)
-        end.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do |_match|
-          full_ref = Regexp.last_match[1]
-          display = Regexp.last_match(2)
-          urn_mention(express_entity_urn(full_ref), display)
-        end
-    end
-
     def entity_name_to_text(entity_id)
       entity_id.downcase.gsub("_", " ")
     end
@@ -425,13 +247,5 @@ module Suma
           "#{entity_name_to_text(entity.id)} #{entity_ref}"
       end
     end
-
-    def convert_express_xref(content, _schema_domain)
-      content.gsub(/<<express:([\w.]+),([\w. ][\w. ]*)>>/) do
-        full_ref = Regexp.last_match(1)
-        display = Regexp.last_match(2)
-        urn_mention(express_entity_urn(full_ref), display)
-      end
-    end
   end
 end

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.4.0"
+  VERSION = "0.5.1"
 end

data/lib/suma.rb

@@ -16,6 +16,7 @@ module Suma
   autoload :LinkValidation,         "suma/link_validation"
   autoload :LinkValidator,          "suma/link_validator"
   autoload :ManifestTraverser,      "suma/manifest_traverser"
+  autoload :NoteProcessor,          "suma/note_processor"
   autoload :RegisterManifestGenerator, "suma/register_manifest_generator"
   autoload :SchemaCategory,         "suma/schema_category"
   autoload :SchemaCollection,       "suma/schema_collection"
@@ -28,6 +29,7 @@ module Suma
   autoload :SchemaNaming,           "suma/schema_naming"
   autoload :SchemaTemplate,         "suma/schema_template"
   autoload :SiteConfig,             "suma/site_config"
+  autoload :StagedCollectionBuilder, "suma/staged_collection_builder"
   autoload :SvgQuality,             "suma/svg_quality"
   autoload :TermClassification,     "suma/term_classification"
   autoload :TermExtractor,          "suma/term_extractor"

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-07-05 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir
@@ -214,7 +213,9 @@ files:
 - lib/suma/jsdai/figure_xml.rb
 - lib/suma/link_validation.rb
 - lib/suma/link_validator.rb
+- lib/suma/link_validator/step.rb
 - lib/suma/manifest_traverser.rb
+- lib/suma/note_processor.rb
 - lib/suma/processor.rb
 - lib/suma/register_manifest_generator.rb
 - lib/suma/schema_category.rb
@@ -230,6 +231,7 @@ files:
 - lib/suma/schema_template/document.rb
 - lib/suma/schema_template/plain.rb
 - lib/suma/site_config.rb
+- lib/suma/staged_collection_builder.rb
 - lib/suma/svg_quality.rb
 - lib/suma/svg_quality/batch_report.rb
 - lib/suma/svg_quality/formatters.rb
@@ -251,7 +253,6 @@ licenses:
 - BSD-2-Clause
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -266,8 +267,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Utility for SUMA (STEP Unified Model-Based Standards Architecture)
 test_files: []