suma 0.5.10 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01e0b1dd212172912b654cac481e0ce1eea1efee1275f45d4332c0a6bbb28e69
-  data.tar.gz: 54399e492be26b0a09595debdc61c256f566da28ed13ca002cae2a6eb0925b57
+  metadata.gz: 7f936716b0ec73649dcebc11349ff94e9979960e0481d7ea48d7d2cfa5edbcfd
+  data.tar.gz: 206887bffa8fb3081bc17a402fccf7bcab927671ffe35c3366ce6b269ab50a15
 SHA512:
-  metadata.gz: 9fb0aebf85233af1df6162efb518605bd7ab01c7396e7af6012144fd05c74a135ba7d2833b7bb633fa45c755e067d6799569412df0b21458b33d2730a5081a3a
-  data.tar.gz: d9836497a01e7711aece5c76534ecfa4e7e0876bfd414aab783ada288b1906e07a3f3b6994d17085f294b79a827c16ecb61bab7cf305a823b9527933e5aa7f1c
+  metadata.gz: af7f5da53bd4cdbf0195381a166334ed51ef5f58434dd3d13cb7c183f0dfe880ce707d3e0d0f75c0a85ab9748cc1b212784f90d30de317997c932fe6811ad985
+  data.tar.gz: d940f6ff6ff164cb4b5a53b1f7c5e45462494d83e3bbf1815b8821e3bb015f5fcb9ec32a8094588dee091cd86b3d0aad4629515e114f5d1399957de22b4bc4df

data/lib/suma/cli/build.rb

@@ -14,8 +14,15 @@ module Suma
                                 desc: "Generate file that contains all schemas in the collection."
       option :staged, type: :boolean, default: false,
                       desc: "Memory-bounded staged build: compile each member in " \
-                            "its own process (sequential) and reinflate. For large " \
-                            "collections that OOM a single-process build (suma#94)."
+                            "its own process (sequential unless --staged-workers " \
+                            "is given) and reinflate. For large collections that " \
+                            "OOM a single-process build (suma#94)."
+      option :staged_workers, type: :numeric, default: 1,
+                              desc: "Concurrent staged render processes (batched " \
+                                    "members per process; requires --staged)"
+      option :staged_batch_size, type: :numeric,
+                                 desc: "Members per staged render process " \
+                                       "(default: 25)"
       option :incremental, type: :boolean, default: false,
                            desc: "Re-render only changed schemas and their " \
                                  "transitive dependents (state file defaults " \
@@ -58,7 +65,6 @@ module Suma
           schemas_all_path: schemas_all_path,
           compile: options[:compile],
           output_directory: "_site",
-          staged: options[:staged],
           incremental: options[:incremental],
           state_file: options[:state_file],
           jobs: options[:jobs],
@@ -66,9 +72,16 @@ module Suma
           sign_method: options[:sign],
           sign_key: options[:sign_key],
           no_valid: options[:no_valid],
+          **staged_options(options),
         ).run
       end
 
+      def staged_options(options)
+        { staged: options[:staged],
+          staged_workers: options[:staged_workers],
+          staged_batch_size: options[:staged_batch_size] }
+      end
+
       def log_error(error)
         Utils.log "[ERROR] Error occurred during processing. See details below."
         Utils.log error

data/lib/suma/closure_validator.rb

@@ -36,8 +36,8 @@ module Suma
     BUILTIN_IDENTIFIERS = %w[
       abs acos asin atan atan2 bexists blength const_e cos exists exp
       format hibound hiindex length lobound loindex log log2 log10 nvl odd
-      roles_of sin sizeof sqrt tan typeof usedin value value_in
-      value_range value_list_by unique
+      roles_of rolesof sin sizeof sqrt tan typeof usedin value value_in
+      value_range value_list_by value_unique unique
       self and or xor not in like equal instanceof
       e pi
     ].freeze
@@ -61,9 +61,9 @@ module Suma
       parsed = manifest.schemas.map do |entry|
         [entry, Expressir::Express::Parser.from_file(entry.path.to_s)]
       end
-      corpus_entities = parsed.flat_map { |_, repo| repo.schemas }
+      corpus_entities = parsed.flat_map { |_, repo| Array(repo.schemas) }
         .flat_map do |schema|
-          schema.children.grep(Expressir::Model::Declarations::Entity)
+          Array(schema.children).grep(Expressir::Model::Declarations::Entity)
         end
         .to_h { |entity| [entity.id.downcase, entity] }
 
@@ -84,10 +84,10 @@ module Suma
     def validate_schema(file, schema, corpus_entities = {})
       violations = []
       scope = SchemaScope.new(schema, graph)
-      entities = schema.children.grep(Expressir::Model::Declarations::Entity)
+      entities = Array(schema.children).grep(Expressir::Model::Declarations::Entity)
         .to_h { |entity| [entity.id.downcase, entity] }
 
-      schema.interfaces.each do |interface|
+      Array(schema.interfaces).each do |interface|
         next if graph.schema_exists?(interface.schema.id)
 
         violations << unresolved(file, schema.id, schema.id, "interface",
@@ -95,7 +95,7 @@ module Suma
                                  "interface target not declared")
       end
 
-      schema.children.each do |owner|
+      Array(schema.children).each do |owner|
         violations.concat(declared_element_references(file, schema.id, owner,
                                                       scope, entities,
                                                       corpus_entities))
@@ -145,7 +145,7 @@ module Suma
       refs = (entity.where_rules || []).flat_map do |rule|
         expression_references(rule.expression, "where", locals)
       end
-      entity.attributes.each do |attribute|
+      Array(entity.attributes).each do |attribute|
         refs.concat(datatype_references(attribute.type, "attribute"))
       end
       refs.concat(supertype_references(entity))
@@ -159,7 +159,7 @@ module Suma
     end
 
     def rule_references(rule)
-      refs = rule.applies_to.map { |ref| ["rule_applies_to", ref.id] }
+      refs = Array(rule.applies_to).map { |ref| ["rule_applies_to", ref.id] }
       locals = Array(rule.variables).filter_map do |variable|
         variable.id&.downcase
       end
@@ -383,24 +383,24 @@ module Suma
         @graph = graph
         @schema_id = schema.id
 
-        schema.children.each { |declaration| add(declaration) }
-        schema.entities.each do |entity|
-          entity.attributes.each { |attribute| add(attribute) }
+        Array(schema.children).each { |declaration| add(declaration) }
+        Array(schema.entities).each do |entity|
+          Array(entity.attributes).each { |attribute| add(attribute) }
         end
-        schema.rules.each do |rule|
-          rule.variables.each { |variable| add(variable) }
-          rule.applies_to.each do |ref|
-            entity = schema.entities.find do |candidate|
+        Array(schema.rules).each do |rule|
+          Array(rule.variables).each { |variable| add(variable) }
+          Array(rule.applies_to).each do |ref|
+            entity = Array(schema.entities).find do |candidate|
               candidate.id&.downcase == ref.id&.downcase
             end
             next unless entity
 
-            entity.attributes.each { |attribute| add(attribute) }
+            Array(entity.attributes).each { |attribute| add(attribute) }
           end
         end
-        schema.constants.each { |constant| add(constant) }
-        schema.functions.each do |function|
-          (function.parameters || []).each { |parameter| add(parameter) }
+        Array(schema.constants).each { |constant| add(constant) }
+        Array(schema.functions).each do |function|
+          Array(function.parameters).each { |parameter| add(parameter) }
         end
       end
 

data/lib/suma/corpus_graph.rb

@@ -134,7 +134,7 @@ module Suma
 
     def declared_elements(schema)
       index = {}
-      schema.children.each do |element|
+      Array(schema.children).each do |element|
         if element.respond_to?(:id) && element.id
           index[normalize(element.id)] =
             true

data/lib/suma/owl_projection.rb

@@ -167,7 +167,7 @@ module Suma
     # the Expressir model and must not shadow declarations).
     def index_elements
       @schemas.each_with_object({}) do |(schema_key, schema), map|
-        schema.children.each do |element|
+        Array(schema.children).each do |element|
           next unless class_element?(element)
 
           map[[schema_key, normalize(element.id)]] = element
@@ -231,7 +231,7 @@ module Suma
     end
 
     def project_schema(graph, schema_key, schema)
-      schema.children.each do |element|
+      Array(schema.children).each do |element|
         case element
         when Expressir::Model::Declarations::Entity
           project_entity(graph, schema_key, element)
@@ -482,7 +482,7 @@ module Suma
 
     def validate_materialization(projected)
       projected.flat_map do |schema_key, schema|
-        schema.children.filter_map do |element|
+        Array(schema.children).filter_map do |element|
           next unless class_element?(element)
 
           iri = element_iri(schema_key, element.id)

data/lib/suma/processor.rb

@@ -13,6 +13,7 @@ module Suma
 
     def initialize(metanorma_yaml_path:, schemas_all_path:, compile: true,
                    output_directory: "_site", staged: false,
+                   staged_workers: 1, staged_batch_size: nil,
                    incremental: false, state_file: nil, jobs: 1,
                    provenance_file: nil, sign_method: nil, sign_key: nil,
                    no_valid: false)
@@ -23,6 +24,8 @@ module Suma
       # Opt-in memory-bounded staged build (metanorma/suma#94); the default
       # single-process build below is unchanged when false.
       @staged = staged
+      @staged_workers = staged_workers
+      @staged_batch_size = staged_batch_size
       @incremental = incremental
       @state_file = state_file
       @jobs = jobs
@@ -181,6 +184,9 @@ module Suma
           output_directory: output_directory,
           coverpage: collection_coverpage(collection_config),
           no_valid: @no_valid,
+          workers: @staged_workers,
+          batch_size: @staged_batch_size ||
+            StagedCollectionBuilder::DEFAULT_BATCH_SIZE,
         ).build
       else
         Metanorma::Collection.parse(COLLECTION_OUTPUT_PATH)

data/lib/suma/schema_exporter.rb

@@ -47,6 +47,18 @@ module Suma
     end
 
     def export_to_directory(schemas)
+      run_pool_export(schemas)
+    rescue SystemStackError
+      # One pathological schema (a very deep AST) can overflow the
+      # worker threads' ~1 MB stack. The main thread has the full OS
+      # stack and every completed export is cached, so the retry only
+      # pays for the schemas that did not finish.
+      Utils.log "[export] worker thread stack exhausted; " \
+                "retrying the remaining schemas on the main thread"
+      schemas.each { |schema| export_single_schema(schema) }
+    end
+
+    def run_pool_export(schemas)
       WorkerPool.new(size: options[:jobs]).map(schemas) do |schema|
         export_single_schema(schema)
       end

data/lib/suma/schema_index.rb

@@ -40,11 +40,11 @@ module Suma
 
     def element_collections(schema)
       [
-        schema.entities,
-        schema.types,
-        schema.constants,
-        schema.functions,
-        schema.rules,
+        Array(schema.entities),
+        Array(schema.types),
+        Array(schema.constants),
+        Array(schema.functions),
+        Array(schema.rules),
         schema.procedures,
         schema.subtype_constraints,
       ]

data/lib/suma/staged_collection_builder.rb

@@ -16,10 +16,12 @@ module Suma
   #      stub-bearing Semantic XML + anchors to a durable content-addressed store
   #      (the metanorma engine's +preserve_unresolved:+ / +artifact_store_dir:+
   #      render options, metanorma/metanorma#576);
-  #   2. runs those per-member processes SEQUENTIALLY (never in parallel --
-  #      parallelism reintroduces the cross-linking / sectionsplit races that are
-  #      deliberately avoided); each process exits and the OS reclaims its memory
-  #      before the next, so peak memory is bounded by a single member; then
+  #   2. runs those per-member processes SEQUENTIALLY by default; with
+  #      +workers+ > 1, batches of members are rendered by that many child
+  #      processes concurrently (safe: members write disjoint store entries,
+  #      and sectionsplit happens only in the sequential reinflate pass); each
+  #      process exit hands its memory back to the OS, so peak memory stays
+  #      bounded by workers x one member; then
   #   3. REINFLATES: a final pass assembles a manifest of the stored stubs and
   #      resolves them into real cross-document links (+reinflate:+), producing
   #      the same output an all-present build would.
@@ -34,12 +36,19 @@ module Suma
   class StagedCollectionBuilder
     STORE_DIRNAME = ".metanorma-collection-cache"
 
+    # Members rendered per child process: one Metanorma boot amortised over
+    # a batch of small members keeps boot cost under a few percent.
+    DEFAULT_BATCH_SIZE = 25
+
     # +collection_config_path+ is the emitted collection manifest (the same
     # +collection-output.yaml+ the normal build renders). Member filerefs are
     # relative to its directory, so all per-member manifests are written there.
-    # (a data-declaring initializer)
+    # +workers+ > 1 runs that many renderer processes concurrently over batches
+    # of +batch_size+ members; members write disjoint, identifier-keyed store
+    # entries, so children cannot race. (a data-declaring initializer)
     def initialize(collection_config_path:, output_directory:, coverpage: nil,
-                   store_dir: nil, formats: %i[xml html], no_valid: false) # rubocop:disable Metrics/ParameterLists
+                   store_dir: nil, formats: %i[xml html], no_valid: false,
+                   workers: 1, batch_size: DEFAULT_BATCH_SIZE) # rubocop:disable Metrics/ParameterLists
       @config_path = collection_config_path
       @config_dir = File.dirname(File.expand_path(collection_config_path))
       @output_directory = output_directory
@@ -47,20 +56,139 @@ module Suma
       @store_dir = store_dir || File.join(output_directory, STORE_DIRNAME)
       @formats = formats
       @no_valid = no_valid
+      @workers = [workers.to_i, 1].max
+      @batch_size = [batch_size.to_i, 1].max
     end
 
     def build
       FileUtils.mkdir_p(@output_directory)
       FileUtils.mkdir_p(@store_dir)
       members = collection_members
-      Utils.log "[staged] #{members.size} members; one process per member, " \
-                "sequential; store: #{@store_dir}"
-      members.each_with_index { |member, i| stage_member(member, i) }
+      Utils.log "[staged] #{members.size} members; workers: #{@workers}; " \
+                "batch size: #{@batch_size}; store: #{@store_dir}"
+      if @workers == 1
+        members.each_with_index { |member, i| stage_member(member, i) }
+      else
+        stage_parallel(members)
+      end
       reinflate(members)
     end
 
     private
 
+    # Batch the members and run +@workers+ renderer child processes
+    # concurrently. Each batch file carries fully-rendered single-member
+    # manifests; children claim batches through O_EXCL markers, so a slow
+    # worker never starves. One Metanorma boot amortises over
+    # +@batch_size+ members (the boot dominates small members), and each
+    # child exit hands its memory back to the OS. Members write disjoint,
+    # identifier-keyed store entries, so concurrent children cannot race;
+    # sectionsplit only happens at reinflate (a single sequential pass).
+    def stage_parallel(members)
+      paths, count = write_batch_files(members)
+      run_batch_workers(count)
+    ensure
+      remove_batch_files(paths, count)
+    end
+
+    def write_batch_files(members)
+      batches = batch_manifests(members)
+      paths = write_batch_bins(batches)
+      Utils.log "[staged] #{batches.size} batches over #{@workers} workers"
+      [paths, batches.size]
+    end
+
+    def batch_manifests(members)
+      members.each_slice(@batch_size).map do |slice|
+        slice.map do |member|
+          base_manifest.merge("manifest" => collection_manifest([member]))
+        end
+      end
+    end
+
+    def write_batch_bins(batches)
+      batches.map.with_index do |batch, i|
+        path = File.join(@config_dir, "._suma_batch_#{i}.bin")
+        File.binwrite(path, Marshal.dump(batch))
+        path
+      end
+    end
+
+    def run_batch_workers(count)
+      children = Array.new([@workers, count].min) do
+        Process.spawn(RbConfig.ruby, "-e", child_script, @config_dir,
+                      count.to_s, @store_dir, @output_directory,
+                      @no_valid ? "1" : "0", chdir: @config_dir)
+      end
+      failed = children
+        .filter_map { |pid| Process.waitpid2(pid) }
+        .map(&:last).reject(&:success?).size
+      return if failed.zero?
+
+      raise "[staged] #{failed} of #{@workers} batch workers failed; " \
+            "see the worker output above"
+    end
+
+    def remove_batch_files(paths, count)
+      paths&.each { |path| FileUtils.rm_f(path) }
+      (0...(count || 0)).each do |i|
+        FileUtils.rm_f(File.join(@config_dir, "._suma_batch_#{i}.claim"))
+      end
+    end
+
+    # Child renderer: claims and renders batches until none remain. The
+    # Metanorma boot is paid once per child, and GC runs between members to
+    # bound the heap. Any render failure exits nonzero and fails the build.
+    def child_script
+      <<~RUBY
+        require "yaml"
+        require "fileutils"
+        require "metanorma"
+        config_dir, batch_count, store_dir, out_dir, no_valid = ARGV
+        count = Integer(batch_count)
+        claimed = 0
+        loop do
+          idx = (0...count).find do |i|
+            begin
+              File.open(File.join(config_dir, "._suma_batch_\#{i}.claim"),
+                        File::RDWR | File::CREAT | File::EXCL) {}
+              true
+            rescue Errno::EEXIST
+              false
+            end
+          end
+          break unless idx
+
+          manifests = Marshal.load(File.binread(File.join(
+            config_dir, "._suma_batch_\#{idx}.bin")))
+          manifests.each_with_index do |manifest, j|
+            one_path = File.join(config_dir,
+                                 "._suma_staged_b\#{idx}_\#{j}.yml")
+            out = File.join(out_dir, "._staged_out_b\#{idx}_\#{j}")
+            t0 = Time.now
+            begin
+              File.write(one_path, YAML.dump(manifest))
+              compile = { install_fonts: false }
+              compile[:novalid] = true if no_valid == "1"
+              Metanorma::Collection.parse(one_path).render(
+                format: [:xml], output_folder: out,
+                preserve_unresolved: true, artifact_store_dir: store_dir,
+                compile: compile,
+              )
+              warn format("[staged worker %d] batch %d/%d member %d OK (%ds)",
+                          Process.pid, idx + 1, count, j, Time.now - t0)
+            ensure
+              File.unlink(one_path) rescue nil
+              FileUtils.rm_rf(out) rescue nil
+              GC.start
+            end
+          end
+          claimed += 1
+        end
+        exit 1 if claimed.zero? && count.positive?
+      RUBY
+    end
+
     def manifest
       @manifest ||= YAML.load_file(@config_path)
     end

data/lib/suma/term_extractor.rb

@@ -94,7 +94,9 @@ module Suma
       section_ref = get_section_ref(schema)
 
       Glossarist::ManagedConceptCollection.new.tap do |collection|
-        schema.entities.each do |entity|
+        # expressir leaves collection attributes nil when the clause is
+        # absent; guard every walk over parsed-model collections.
+        Array(schema.entities).each do |entity|
           managed_data = build_managed_data(schema, entity, section_ref,
                                             classification)
 
@@ -154,7 +156,7 @@ module Suma
         data.sources = [build_source_ref(schema, entity)]
 
         notes = NoteProcessor.call(
-          entity.remarks,
+          Array(entity.remarks),
           definitions: data.definition,
           xref_to_mention: method(:xref_to_mention),
         )
@@ -273,12 +275,16 @@ module Suma
 
       entity_ref = urn_mention(term_urn("express-language.entity"), "entity")
 
-      if entity.subtype_of.empty?
+      # expressir 2.4.44 returns nil for entities without SUBTYPE_OF
+      # (older versions returned []); guard both shapes.
+      subtypes = entity.subtype_of || []
+
+      if subtypes.empty?
         "#{entity_type} " \
           "that represents the " \
           "#{entity_name_to_text(entity.id)} #{entity_ref}"
       else
-        entity_subtypes = entity.subtype_of.map do |e|
+        entity_subtypes = subtypes.map do |e|
           urn_mention(express_entity_urn("#{schema.id}.#{e.id}"), e.id)
         end
 

data/lib/suma/urn_resolver.rb

@@ -96,7 +96,7 @@ module Suma
       artifacts = {}
       schemas.each do |schema|
         artifacts[compose(:tech, schema.id)] = schema_payload(schema)
-        schema.children.each do |element|
+        Array(schema.children).each do |element|
           payload = element_payload(schema, element)
           artifacts[compose(:tech, "#{schema.id}.#{element.id}")] = payload if
             payload
@@ -111,7 +111,7 @@ module Suma
     def add_attribute_artifacts(artifacts, schema, element)
       return unless element.is_a?(Expressir::Model::Declarations::Entity)
 
-      element.attributes.each do |attribute|
+      Array(element.attributes).each do |attribute|
         key = compose(:tech,
                       "#{schema.id}.#{element.id}.#{attribute.id}")
         artifacts[key] = attribute_artifact(schema, element, attribute)
@@ -150,7 +150,7 @@ module Suma
         "interface" => Array(schema.interfaces).filter_map do |interface|
           interface.schema&.id
         end,
-        "elements" => schema.children.filter_map do |element|
+        "elements" => Array(schema.children).filter_map do |element|
           element.id if declared_element?(element)
         end.sort,
       }

data/lib/suma/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Suma
-  VERSION = "0.5.10"
+  VERSION = "0.5.11"
 end

data/lib/suma/worker_pool.rb

@@ -45,7 +45,11 @@ module Suma
           item, index = pair
           begin
             results[index] = block.call(item)
-          rescue StandardError => e
+          # SystemStackError is not StandardError: deep per-item recursion
+          # (a pathological EXPRESS AST in the export pretty-printer)
+          # overflows the ~1 MB thread stack, and swallowing it would
+          # leave silent holes in results. The caller falls back.
+          rescue SystemStackError, StandardError => e
             errors << e
             break
           end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: suma
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-10-21 00:00:00.000000000 Z
+date: 2026-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: expressir