metanorma-plugin-lutaml 0.7.53 → 0.7.54

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05df720264c4dabcd31cb6957904853bc928cfe2e8cb49eec614dfa8d658eafd
-  data.tar.gz: 0da436fb95370e1a7d0fa4241a65086d00eb2a6496d314460bc54499d9a0fa1c
+  metadata.gz: b6a443f1152ff2cf6f053744b7750e0e96426bc776cf0cb582b9c93f7d828f55
+  data.tar.gz: 4f31cad3531f1f38525e2eaacee72917fee007a2a6099a772894ed6159ed5168
 SHA512:
-  metadata.gz: '028f5021f95d108c7c55efba46e5a8bad6558690593b636c71d244bf7896ad639d10176853551127256b0d0206993ebf8587700786de846dacbad547ce066c39'
-  data.tar.gz: c7136a9bd3a6937583aaa6f04e03e3867fbbb9cc18b1715cf91ce369329badd4a7ea0e602b5c1ae3d4303a5f14b9838786ee2d605d821ed70bc22401de15ce2e
+  metadata.gz: 4098ac9e010d91d65a98c21eb22abcbd7add8166f7dc93a1f0b2fdeeb790b40c70a274ef8b66f36aaa3402bcc3b481d9283ff4e766586b4a33e4c8dfd75e2463
+  data.tar.gz: 9ed735f7bd7b870a8be1ad0847e4b9c47a2a411a23de2a7d98874c6d185286e60123d067c8e34be42b3d339ae021cee51190b50c8c0581a5a04c1ad47c64f654

data/.github/workflows/rake.yml

@@ -20,30 +20,5 @@ jobs:
       setup-tools: graphviz
       submodules: true
       choco-cache: true
-      # Build the expressir Rust binding for the vendored gem: without
-      # it the pure-Ruby parser path makes the heavy suites exceed the
-      # runner memory ceiling on ruby 3.3 (#304).
-      after-setup-ruby: |
-        case "$(uname -s)" in
-          *MINGW*|*MSYS*)
-            echo "windows: ruby is mingw-built; the binding needs the
-            gnu toolchain - skipping (windows passes without it)"
-            exit 0
-            ;;
-        esac
-        if ! command -v cargo >/dev/null 2>&1; then
-          echo "cargo not available; skipping expressir binding build"
-          exit 0
-        fi
-        gem install rb_sys --no-document
-        gem_path=$(bundle info expressir --path)
-        ext_dir="$gem_path/ext/expressir_core"
-        if [ -f "$ext_dir/extconf.rb" ]; then
-          (cd "$ext_dir" && ruby extconf.rb && make)
-          bundle_path="$ext_dir/expressir_core.bundle"
-          if [ "$(uname)" = "Darwin" ] && [ -f "$bundle_path" ]; then
-            codesign -f -s - "$bundle_path"
-          fi
-        fi
     secrets:
       pat_token: ${{ secrets.METANORMA_CI_PAT_TOKEN }}

data/lib/metanorma/plugin/lutaml/gc_budget.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Plugin
+    module Lutaml
+      # The lutaml macro family churns transients of hundreds of MB per
+      # invocation on top of a multi-GB live parsed-XMI set. The churn is
+      # malloc-side (libxml/moxml/liquid), so the GC's heap-growth
+      # heuristics fire too late under memory caps; collect at macro
+      # boundaries whenever uncollected malloc has exceeded the budget.
+      module GcBudget
+        BUDGET_BYTES = 512 * (1 << 20)
+
+        class << self
+          def gc_when_bloated!
+            GC.start if GC.stat(:malloc_increase_bytes) > BUDGET_BYTES
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/lutaml/liquid_templates/_klass_table.liquid

@@ -1,4 +1,4 @@
-{% assign root = klass.generalization %}
+{%- assign root = klass.generalization -%}
 
 {%- capture upper_klass_name -%}
 {{ root.general.upper_klass }}:{{ root.general.name }}
@@ -21,57 +21,57 @@ h| Stereotype 2+| {{ stereotype }}
 
 3+h| Inherited Properties
 h| Property Name h| Property Type and Multiplicity h| Definition
-{% for attr in root.inherited_props %}
-  {%- unless attr.has_association? -%}
-    {%- capture name_col -%}
+{%- for attr in root.inherited_props -%}
+{%- unless attr.has_association? -%}
+{%- capture name_col -%}
       {{ attr.name_ns }}:{{ attr.name }}
       ({{ attr.gen_name }})
     {%- endcapture -%}
     | {{ name_col | newline_to_br }}
     | {{ attr.type }} [{{ attr.cardinality.min }}..{{ attr.cardinality.max }}]
     | {{ attr.definition }}
-  {%- endunless -%}
-{% endfor %}
+{%- endunless -%}
+{%- endfor -%}
 
 3+h| Self-defined Properties
 h| Property Name h| Property Type and Multiplicity h| Definition
-{% for attr in root.owned_props %}
-  {%- unless attr.has_association? -%}
-    {%- capture name_col -%}
+{%- for attr in root.owned_props -%}
+{%- unless attr.has_association? -%}
+{%- capture name_col -%}
       {{ attr.name_ns }}:{{ attr.name }}
       ({{ attr.gen_name }})
     {%- endcapture -%}
     | {{ name_col | newline_to_br }}
     | {{ attr.type }} [{{ attr.cardinality.min }}..{{ attr.cardinality.max }}]
     | {{ attr.definition }}
-  {%- endunless -%}
-{% endfor %}
+{%- endunless -%}
+{%- endfor -%}
 
 3+h| Properties Inherited from Association
 h| Property Name h| Property Type and Multiplicity h| Definition
-{% for attr in root.inherited_assoc_props %}
-  {%- if attr.has_association? -%}
-    {%- capture name_col -%}
+{%- for attr in root.inherited_assoc_props -%}
+{%- if attr.has_association? -%}
+{%- capture name_col -%}
       {{ attr.name_ns }}:{{ attr.name }}
       ({{ attr.gen_name }})
     {%- endcapture -%}
     | {{ name_col | newline_to_br }}
     | {{ attr.type_ns }}:{{ attr.type }} [{{ attr.cardinality.min }}..{{ attr.cardinality.max }}]
     | {{ attr.definition }}
-  {%- endif -%}
-{% endfor %}
+{%- endif -%}
+{%- endfor -%}
 
 3+h| Properties Defined in Association
 h| Property Name h| Property Type and Multiplicity h| Definition
-{% for attr in root.assoc_props %}
-  {%- if attr.has_association? -%}
-    {%- capture name_col -%}
+{%- for attr in root.assoc_props -%}
+{%- if attr.has_association? -%}
+{%- capture name_col -%}
       {{ attr.name_ns }}:{{ attr.name }}
       ({{ attr.gen_name }})
     {%- endcapture -%}
     | {{ name_col | newline_to_br }}
     | {{ attr.type_ns }}:{{ attr.type }} [{{ attr.cardinality.min }}..{{ attr.cardinality.max }}]
     | {{ attr.definition }}
-  {%- endif -%}
-{% endfor %}
-|===
+{%- endif -%}
+{%- endfor -%}
+|===

data/lib/metanorma/plugin/lutaml/lutaml_data_preprocessor.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Plugin
+    module Lutaml
+      # Renders LutaML data instances through a Liquid template.
+      #
+      # Data definitions and instances in separate files:
+      #
+      #   [lutaml_data,<definitions.lml>,<instances.lml|yaml|json>,<context>]
+      #   ----
+      #   {% for check in context.S158Checks.checks %}
+      #   {{ check.dev_id }}
+      #   {% endfor %}
+      #   ----
+      #
+      # Data definitions and instances in the same file:
+      #
+      #   [lutaml_data,<definitions.lml>,<context>]
+      #
+      class LutamlDataPreprocessor < BasePreprocessor
+        DATA_FILE_EXTENSIONS = %w[.lml .yaml .yml .json].freeze
+        DEFAULT_CONTEXT_NAME = "lutaml_data"
+
+        DATA_BLOCK_REGEX = %r{
+          ^
+          \[
+          \blutaml_data\b
+          ,(?<definition_file>[^,\[\]]+)
+          (?<rest>.*)
+          \]
+        }x
+
+        protected
+
+        def lutaml_liquid?(line)
+          line.match(DATA_BLOCK_REGEX)
+        end
+
+        def index_type_name
+          "LutaML data"
+        end
+
+        def load_lutaml_file(_document, file_path, _options)
+          ::Lutaml::Lml.parse_document(File.new(file_path, encoding: "UTF-8"))
+        end
+
+        private
+
+        def process_text_blocks(document, input_lines, _express_indexes)
+          line = input_lines.next
+          block_header_match = lutaml_liquid?(line)
+          return [line] unless block_header_match
+
+          definition_file, instance_file, context_name, options =
+            parse_header(block_header_match)
+
+          end_mark = input_lines.next
+          lines = extract_block_lines(input_lines, end_mark)
+
+          render_data_template(
+            document: document,
+            lines: lines,
+            definition_file: definition_file,
+            instance_file: instance_file,
+            context_name: context_name,
+            options: options,
+          )
+        end
+
+        def parse_header(match)
+          definition_file = match[:definition_file].strip
+          segments = match[:rest].split(",").map(&:strip)
+          options, segments = split_options(segments)
+
+          data_segments, name_segments = segments
+            .reject(&:empty?).partition { |segment| data_file?(segment) }
+
+          [definition_file, data_segments.last,
+           name_segments.last || DEFAULT_CONTEXT_NAME, options]
+        end
+
+        def split_options(segments)
+          options_start = segments.index { |segment| segment.include?("=") }
+          return [{}, segments] unless options_start
+
+          options = parse_options(",#{segments[options_start..].join(',')}")
+          [options, segments[0...options_start]]
+        end
+
+        def data_file?(segment)
+          DATA_FILE_EXTENSIONS.include?(File.extname(segment))
+        end
+
+        def render_data_template(document:, lines:, # rubocop:disable Metrics/ParameterLists
+                                 definition_file:, instance_file:,
+                                 context_name:, options:)
+          definition_path, instances_path = resolve_data_paths(
+            document, definition_file, instance_file
+          )
+
+          compiler = ::Lutaml::Lml::ModelCompiler.new
+          compiler.compile(File.new(definition_path, encoding: "UTF-8"))
+          instances = load_instances(instances_path, compiler)
+
+          [render_liquid(lines, document, options, context_name, instances)]
+        rescue StandardError => e
+          ::Metanorma::Util.log(
+            "[#{self.class.name}] Failed to parse LutaML data block: " \
+            "#{e.message}",
+            :error,
+          )
+          raise e
+        end
+
+        def render_liquid(lines, document, options, context_name, instances)
+          parsed_template = template(lines)
+          parsed_template.registers[:file_system] =
+            build_file_system(document, options)
+          parsed_template.render(context_name => instances.to_h)
+        end
+
+        def resolve_data_paths(document, definition_file, instance_file)
+          definition_path = Utils.relative_file_path(document, definition_file)
+          unless File.file?(definition_path)
+            raise StandardError, index_missing_message(definition_file)
+          end
+
+          instances_path = instance_file &&
+            Utils.relative_file_path(document, instance_file)
+          if instances_path && !File.file?(instances_path)
+            raise StandardError, index_missing_message(instance_file)
+          end
+
+          [definition_path, instances_path || definition_path]
+        end
+
+        def load_instances(path, compiler)
+          case File.extname(path)
+          when ".lml" then load_lml_instances(path, compiler)
+          when ".json" then load_structured_instances(path, compiler,
+                                                      :parse_json_data)
+          else load_structured_instances(path, compiler, :parse_yaml_data)
+          end
+        end
+
+        def load_lml_instances(path, compiler)
+          doc = load_lutaml_file(nil, path, {})
+          root = doc.instance
+          return [] unless root
+
+          [[root.type.to_s, compiler.hydrate(doc)]]
+        rescue StandardError => e
+          raise StandardError, "Failed to parse LutaML instances in " \
+                               "`#{path}`: #{e.message}"
+        end
+
+        def load_structured_instances(path, compiler, parser)
+          data = send(parser, path)
+          entries = data.is_a?(Array) ? data : [data]
+          entries.map { |entry| build_structured_entry(path, compiler, entry) }
+        rescue StandardError => e
+          raise e if e.message.start_with?("Instance", "Unknown type")
+
+          raise StandardError, "Failed to load data instances in " \
+                               "`#{path}`: #{e.message}"
+        end
+
+        def build_structured_entry(path, compiler, entry)
+          type = entry.delete("type")
+          unless type
+            raise StandardError, "Instance in `#{path}` is missing the " \
+                                 "`type` key"
+          end
+
+          klass = compiler.compiled_classes[demodulize(type)]
+          unless klass
+            raise StandardError,
+                  "Unknown type `#{type}` in `#{path}`"
+          end
+
+          [demodulize(type), klass.from_hash(entry)]
+        end
+
+        def parse_yaml_data(path)
+          require "yaml"
+          YAML.safe_load(File.read(path, encoding: "UTF-8"))
+        end
+
+        def parse_json_data(path)
+          require "json"
+          JSON.parse(File.read(path, encoding: "UTF-8"))
+        end
+
+        def demodulize(name)
+          name.to_s.split("::").last
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/lutaml/lutaml_gml_dictionary_base.rb

@@ -3,6 +3,8 @@
 require "ogc/gml"
 require "liquid"
 
+require "metanorma/plugin/lutaml/gc_budget"
+
 module Metanorma
   module Plugin
     module Lutaml
@@ -10,6 +12,7 @@ module Metanorma
         private
 
         def render(tmpl, parent, attrs, orig_gml_path)
+          GcBudget.gc_when_bloated!
           dict = get_gml_dictionary(parent, orig_gml_path)
           tmpl.assigns[attrs["context"]] = GmlDictionaryDrop.new(dict)
           rendered_tmpl = tmpl.render
@@ -17,12 +20,19 @@ module Metanorma
           parse_content(block, rendered_tmpl, attrs)
         end
 
+        # A document-scale compile resolves the same dictionary files many
+        # times (the plateau corpus cites codelists up to 34 times each);
+        # re-reading and re-parsing per macro was pure duplicate work.
+        GML_DICTIONARY_CACHE = CacheStore.new(max_size: 512)
+
         def get_gml_dictionary(parent, orig_gml_path)
           gml_path = Utils.relative_file_path(
             parent.document, orig_gml_path
           )
 
-          ::Ogc::Gml::Dictionary.from_xml(xml_content(gml_path))
+          GML_DICTIONARY_CACHE.fetch_or_store(gml_path) do
+            ::Ogc::Gml::Dictionary.from_xml(xml_content(gml_path))
+          end
         end
 
         def xml_content(filepath)

data/lib/metanorma/plugin/lutaml/lutaml_klass_table_block_macro.rb

@@ -28,8 +28,8 @@ module Metanorma
             guidance = get_guidance(parent.document, attrs["guidance"])
           end
 
-          klass = serialize_klass_drop_by_name(xmi_path, path, parent.document,
-                                               guidance)
+          klass = serialize_klass_or_datatype_drop_by_name(
+            xmi_path, path, parent.document, guidance)
 
           render_table(klass, CONTEXT_NAME, parent, attrs)
         end

data/lib/metanorma/plugin/lutaml/version.rb

@@ -1,7 +1,7 @@
 module Metanorma
   module Plugin
     module Lutaml
-      VERSION = "0.7.53".freeze
+      VERSION = "0.7.54".freeze
     end
   end
 end

data/lib/metanorma/plugin/lutaml/xmi_cache.rb

@@ -1,10 +1,17 @@
 # frozen_string_literal: true
 
+require "metanorma/plugin/lutaml/gc_budget"
+
 module Metanorma
   module Plugin
     module Lutaml
       module XmiCache
-        XMI_PARSE_CACHE = XmiParseCache.new
+        # Bounded parse-once cache: every macro touching a source (the
+        # full file, or its package slice under lutaml-ea-xmi-load: partial)
+        # reuses the one parsed graph; the LRU bound keeps the retained
+        # graphs to a few live at once, which the section-ordered
+        # macros of a document turn into near-perfect reuse.
+        XMI_PARSE_CACHE = XmiParseCache.new(max_size: 4)
 
         def lutaml_document_from_file_or_cache(document, file_path, yaml_config,
 yaml_config_path = nil)
@@ -27,21 +34,8 @@ yaml_config_path = nil)
           end
         end
 
-        def find_class_by_xmi_id(container, xmi_id)
-          container.classes.find { |node| node.xmi_id == xmi_id } ||
-            container.packages
-              .lazy
-              .filter_map { |pkg| find_class_by_xmi_id(pkg, xmi_id) }
-              .first
-        end
 
-        def find_enum_by_xmi_id(container, xmi_id)
-          container.enums.find { |node| node.xmi_id == xmi_id } ||
-            container.packages
-              .lazy
-              .filter_map { |pkg| find_enum_by_xmi_id(pkg, xmi_id) }
-              .first
-        end
+
 
         def find_packaged_klass(index, path, root_model_name: nil)
           segments = path.split("::").reject(&:empty?)
@@ -69,6 +63,32 @@ yaml_config_path = nil)
           end
         end
 
+        def find_packaged_datatype(index, path, root_model_name: nil)
+          segments = path.split("::").reject(&:empty?)
+          if root_model_name && segments.first == root_model_name
+            segments.shift
+          end
+          if segments.one?
+            index.find_packaged_by_name_and_types(
+              segments.first, ["uml:DataType"]
+            )
+          else
+            find_packaged_datatype_by_path(index, segments)
+          end
+        end
+
+        def find_packaged_datatype_by_path(index, segments)
+          datatype_name = segments.pop
+
+          candidates = ["uml:DataType"]
+            .flat_map { |t| index.packaged_elements_of_type(t) }
+            .select { |e| e.name == datatype_name }
+
+          candidates.find do |datatype|
+            match_parent_chain?(index, datatype, segments)
+          end
+        end
+
         def match_parent_chain?(index, element, parent_segments)
           current = element
           parent_segments.reverse_each do |pkg_name|
@@ -85,23 +105,54 @@ yaml_config_path = nil)
             .find { |e| e.name == name }
         end
 
-        def serialize_klass_drop_by_name(xmi_path, name, _document = nil,
-guidance = nil)
-          parsed = XMI_PARSE_CACHE.fetch(xmi_path)
+        def serialize_klass_or_datatype_drop_by_name(xmi_path,
+          name, _document = nil, guidance = nil)
+          serialize_klass_drop_by_name(xmi_path, name, _document, guidance) ||
+            serialize_datatype_drop_by_name(xmi_path, name, _document)
+        end
+
+        def serialize_klass_drop_by_name(xmi_path,
+          name, _document = nil, guidance = nil)
+          GcBudget.gc_when_bloated!
+          xmi_path, name, slice = XmiPartialLoadRegistry.rewrite(xmi_path, name)
+          parsed = XMI_PARSE_CACHE.fetch(xmi_path, slice: slice)
           klass = resolve_packaged_klass(parsed, name)
-          warn "Class not found for name: #{name}" if klass.nil?
+          if klass.nil?
+            warn "Class not found for name: #{name}"
+            return nil
+          end
+
           ::Ea::Xmi::LiquidDrops::KlassDrop.new(
             klass, guidance, parsed.drop_options
           )
         end
 
+        def serialize_datatype_drop_by_name(xmi_path, name, _document = nil)
+          GcBudget.gc_when_bloated!
+          xmi_path, name, slice = XmiPartialLoadRegistry.rewrite(xmi_path, name)
+          parsed = XMI_PARSE_CACHE.fetch(xmi_path, slice: slice)
+          datatype = resolve_packaged_datatype(parsed, name)
+          if datatype.nil?
+            warn "Datatype not found for name: #{name}"
+            nil
+          end
+
+          ::Ea::Xmi::LiquidDrops::DataTypeDrop.new(
+            datatype, parsed.drop_options
+          )
+        end
+
         def serialize_enum_drop_by_name(xmi_path, name, _document = nil)
-          parsed = XMI_PARSE_CACHE.fetch(xmi_path)
+          GcBudget.gc_when_bloated!
+          xmi_path, name, slice = XmiPartialLoadRegistry.rewrite(xmi_path, name)
+          parsed = XMI_PARSE_CACHE.fetch(xmi_path, slice: slice)
           raw_enum = find_packaged_enum(parsed.parser.xmi_index, name)
-          warn "Enumeration not found for name: #{name}" if raw_enum.nil?
-          enum = raw_enum && find_enum_by_xmi_id(
-            parsed.uml_document, raw_enum.id
-          )
+          if raw_enum.nil?
+            warn "Enumeration not found for name: #{name}"
+            return nil
+          end
+
+          enum = parsed.xmi_id_index[raw_enum.id]
           ::Ea::Xmi::LiquidDrops::EnumDrop.new(
             enum, parsed.drop_options
           )
@@ -115,9 +166,16 @@ guidance = nil)
             parsed.parser.xmi_index, name,
             root_model_name: root_model_name
           )
-          raw_klass && find_class_by_xmi_id(
-            parsed.uml_document, raw_klass.id
+          raw_klass && parsed.xmi_id_index[raw_klass.id]
+        end
+
+        def resolve_packaged_datatype(parsed, name)
+          root_model_name = parsed.parser.xmi_root_model.model.name
+          raw_datatype = find_packaged_datatype(
+            parsed.parser.xmi_index, name,
+            root_model_name: root_model_name
           )
+          raw_datatype && parsed.xmi_id_index[raw_datatype.id]
         end
       end
     end

data/lib/metanorma/plugin/lutaml/xmi_parse_cache.rb

@@ -1,36 +1,36 @@
 # frozen_string_literal: true
 
+require "stringio"
 require "xmi"
 require "ea/xmi"
 
 module Metanorma
   module Plugin
     module Lutaml
-      ParsedXmi = Struct.new(:parser, :uml_document, :drop_options,
-                             keyword_init: true)
-
       class XmiParseCache
         def initialize(max_size: 50)
           @parse_cache = CacheStore.new(max_size: max_size)
           @drop_cache = CacheStore.new(max_size: max_size)
         end
 
-        def fetch(full_path)
-          @parse_cache.fetch_or_store(full_path) do
-            xmi_model = ::Xmi::Sparx::Root.parse_xml(File.read(full_path))
-            parser = ::Ea::Xmi::Parser.new
-            uml_document = parser.parse(xmi_model)
-            ParsedXmi.new(
-              parser: parser,
-              uml_document: uml_document,
-              drop_options: build_drop_options(parser),
-            )
+        # slice: a XmiPartialLoadRegistry token; when given, the macro parses
+        # the package slice prepared for it instead of the whole source
+        def fetch(full_path, retain: true, slice: nil)
+          key = slice || full_path
+          return fetch_unretained(full_path, slice) unless retain
+
+          @parse_cache.fetch_or_store(key) do
+            load_parsed(full_path, slice)
           end
         end
 
-        def fetch_drop(full_path, guidance: nil)
-          parsed = fetch(full_path)
-          @drop_cache.fetch_or_store([full_path, guidance]) do
+        def fetch_unretained(full_path, slice = nil)
+          load_parsed(full_path, slice)
+        end
+
+        def fetch_drop(full_path, guidance: nil, slice: nil)
+          parsed = fetch(full_path, slice: slice)
+          @drop_cache.fetch_or_store([key_for(full_path, slice), guidance]) do
             ::Ea::Xmi::LiquidDrops::RootDrop.new(
               parsed.uml_document, guidance, parsed.drop_options
             )
@@ -44,16 +44,20 @@ module Metanorma
 
         private
 
-        def build_drop_options(parser)
-          lookup = ::Ea::Xmi::LookupService.new(parser)
-          {
-            xmi_root_model: parser.xmi_root_model,
-            id_name_mapping: parser.id_name_mapping,
-            lookup: lookup,
-            with_gen: true,
-            with_assoc: true,
-            with_absolute_path: true,
-          }
+        def key_for(full_path, slice)
+          slice || full_path
+        end
+
+        # The prepared slice is parsed whole: assembling it already
+        # streamed the source once, so nothing re-reads the original
+        # export here.
+        def load_parsed(full_path, slice)
+          if slice
+            xml = XmiPartialLoadRegistry.slice_string(slice)
+            ::Ea::Xmi.load_graph(StringIO.new(xml))
+          else
+            ::Ea::Xmi.load_graph(full_path)
+          end
         end
       end
     end

data/lib/metanorma/plugin/lutaml/xmi_partial_load_registry.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "pathname"
+require_relative "utils"
+
+module Metanorma
+  module Plugin
+    module Lutaml
+      # Loads the XMI sources of table macros partially, through
+      # Ea::Xmi.load_graph's partial mode.
+      #
+      # EA exports of plateau scale reach ~100 MB, more than half of it
+      # diagram presentation data, and fully materializing the Xmi tree
+      # plus the Ea graph costs 10+ GB of live objects - more than a
+      # standard build runner has. With the `lutaml-ea-xmi-load` document
+      # attribute set to `partial`, every klass and enum table macro's
+      # (package, name) pair seeds a per-package reference closure; one
+      # streaming pass per source assembles the closures in memory and
+      # each macro parses only its own package's slice. Nothing is
+      # written to disk.
+      #
+      # Without the attribute nothing is preloaded and behavior is
+      # byte-for-byte the whole-load path.
+      module XmiPartialLoadRegistry
+        MACRO_REGEXP = /lutaml_(?:klass|enum)_table::([^\[<]+)\[([^\]]*)\]/
+        MIN_SOURCE_BYTES = 2 * 1024 * 1024
+
+        class << self
+          # Rewrites (xmi_path, name_path) to the partial-load form of
+          # the macro's class: [path, bare name, slice token]. The
+          # token is nil when no slice was prepared, in which case the
+          # macro whole-loads the source unchanged.
+          def rewrite(xmi_path, name_path)
+            return [xmi_path, name_path, nil] unless @registry
+
+            entry = @registry[xmi_path]
+            return [xmi_path, name_path, nil] unless entry
+
+            group_key = entry[name_path] || entry[bare_name(name_path)]
+            return [xmi_path, name_path, nil] unless group_key
+
+            [xmi_path, bare_name(name_path), [xmi_path, group_key]]
+          end
+
+          # The prepared slice for a token from #rewrite.
+          def slice_string(token)
+            @slices[token]
+          end
+
+          def slice?(xmi_path)
+            return false unless @registry
+
+            entry = @registry[xmi_path]
+            !entry.nil? && entry.any?
+          end
+
+          def reset!
+            @registry = nil
+            @slices = nil
+          end
+
+          def prepare(document, lines) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+            reset!
+            # header attributes are parsed after preprocessors run, so
+            # the opt-in is also detected in the raw lines
+            return unless enabled?(document, lines)
+
+            groups = collect_groups(document, lines)
+            return if groups.empty?
+
+            @registry = {}
+            @slices = {}
+            groups.each do |path, entries|
+              next unless File.size(path) >= MIN_SOURCE_BYTES
+
+              slice_groups = {}
+              macro_keys = {}
+              entries.each do |key, value|
+                if value.is_a?(Array) # group_key => wanted pairs
+                  slice_groups[key] = value
+                else # macro key => group_key
+                  (macro_keys[value] ||= []) << key
+                end
+              end
+              # one index pass and one write pass over the source; the
+              # slices stay in memory for the parse cache to consume,
+              # so nothing is written to disk
+              ::Ea::Xmi::Slicer.slices(path, slice_groups).each do |group_key, xml|
+                token = [path, group_key]
+                @slices[token] = xml
+                macro_keys[group_key]&.each do |k|
+                  (@registry[path] ||= {})[k] = group_key
+                end
+              end
+            end
+            @registry = nil if @registry.empty?
+          end
+
+          private
+
+          def enabled?(document, lines)
+            document.attributes["lutaml-ea-xmi-load"] == "partial" ||
+              lines.any? { |l| l.match?(/\A:lutaml-ea-xmi-load:\s*partial\s*$/) }
+          end
+
+          # Slices are grouped per package, not per class: the tables
+          # of classes in one package share most of their closure
+          # (partners, connectors, ancestors), so a package slice
+          # carries each shared subtree once instead of once per class.
+          def collect_groups(document, lines)
+            groups = Hash.new { |h, k| h[k] = {} }
+            lines.each do |line|
+              MACRO_REGEXP.match(line) do |m|
+                path = Utils.relative_file_path(document, m[1].strip)
+                package = m[2][/&?(?:^|,)package="([^"]+)"/, 1]
+                name = m[2][/&?(?:^|,)name="([^"]+)"/, 1]
+                next if name.nil?
+
+                key = package ? "#{package}::#{name}" : name
+                group_key = package || "_"
+                (groups[path][group_key] ||= []) << [package, name]
+                groups[path][key] = group_key
+              end
+            end
+            groups
+          end
+
+          def bare_name(name_path)
+            name_path.to_s.split("::").last
+          end
+        end
+      end
+
+      # Scans the fully expanded document source once, before parsing,
+      # and prepares the partial-load seeds for every table macro. The
+      # lines pass through unchanged; reading them through the incoming
+      # reader expands include directives, which is where plateau
+      # documents keep their table macros.
+      class XmiPartialLoadPreprocessor < ::Asciidoctor::Extensions::Preprocessor
+        def process(document, reader)
+          input_lines = reader.readlines
+          XmiPartialLoadRegistry.prepare(document, input_lines)
+          reader.class.new(document, input_lines)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/plugin/lutaml/xmi_renderer.rb

@@ -2,6 +2,8 @@
 
 require "liquid"
 
+require "metanorma/plugin/lutaml/gc_budget"
+
 module Metanorma
   module Plugin
     module Lutaml
@@ -72,6 +74,9 @@ module Metanorma
           end
 
           rendered_table = table_tmpl.render
+          # The asciidoctor ingestion of a rendered table transiently
+          # dwarfs its input; collect the liquid phase's garbage first.
+          GcBudget.gc_when_bloated!
           block = create_open_block(parent, "", attrs)
           parse_content(block, rendered_table, attrs)
         end

data/lib/metanorma-plugin-lutaml.rb

@@ -31,6 +31,8 @@ module Metanorma
       autoload :LutamlDiagramBase, "metanorma/plugin/lutaml/lutaml_diagram_base"
       autoload :LutamlDiagramBlock,
                "metanorma/plugin/lutaml/lutaml_diagram_block"
+      autoload :LutamlDataPreprocessor,
+               "metanorma/plugin/lutaml/lutaml_data_preprocessor"
       autoload :LutamlDiagramBlockMacro,
                "metanorma/plugin/lutaml/lutaml_diagram_block_macro"
       autoload :LutamlEaDiagramBlockMacro,
@@ -65,6 +67,10 @@ module Metanorma
                "metanorma/plugin/lutaml/lutaml_xsd_preprocessor"
       autoload :ParseError, "metanorma/plugin/lutaml/parse_error"
       autoload :XmiParseCache, "metanorma/plugin/lutaml/xmi_parse_cache"
+      autoload :XmiPartialLoadRegistry,
+               "metanorma/plugin/lutaml/xmi_partial_load_registry"
+      autoload :XmiPartialLoadPreprocessor,
+               "metanorma/plugin/lutaml/xmi_partial_load_registry"
       autoload :SourceExtractor, "metanorma/plugin/lutaml/source_extractor"
       autoload :Utils, "metanorma/plugin/lutaml/utils"
       autoload :Yaml2TextPreprocessor,

data/metanorma-plugin-lutaml.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "asciidoctor"
   spec.add_dependency "coradoc", "~> 1.1.8"
-  spec.add_dependency "ea"
+  spec.add_dependency "ea", ">= 0.6.41" # Ea::Xmi.load/load_graph (lutaml/ea#86): the partial-load API this gem calls
   spec.add_dependency "expressir", "~> 2.3", ">= 2.3.5"
   spec.add_dependency "isodoc"
   spec.add_dependency "liquid"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-plugin-lutaml
 version: !ruby/object:Gem::Version
-  version: 0.7.53
+  version: 0.7.54
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-26 00:00:00.000000000 Z
+date: 2026-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.41
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.6.41
 - !ruby/object:Gem::Dependency
   name: expressir
   requirement: !ruby/object:Gem::Requirement
@@ -224,6 +224,7 @@ files:
 - lib/metanorma/plugin/lutaml/express_remark_adapter.rb
 - lib/metanorma/plugin/lutaml/express_remarks_decorator.rb
 - lib/metanorma/plugin/lutaml/file_not_found_error.rb
+- lib/metanorma/plugin/lutaml/gc_budget.rb
 - lib/metanorma/plugin/lutaml/json2_text_preprocessor.rb
 - lib/metanorma/plugin/lutaml/liquid.rb
 - lib/metanorma/plugin/lutaml/liquid/custom_blocks.rb
@@ -250,6 +251,7 @@ files:
 - lib/metanorma/plugin/lutaml/liquid_templates/_packages_entity_list_class.liquid
 - lib/metanorma/plugin/lutaml/liquid_templates/_packages_entity_list_classes.liquid
 - lib/metanorma/plugin/lutaml/liquid_templates/_packages_enum.liquid
+- lib/metanorma/plugin/lutaml/lutaml_data_preprocessor.rb
 - lib/metanorma/plugin/lutaml/lutaml_diagram_base.rb
 - lib/metanorma/plugin/lutaml/lutaml_diagram_block.rb
 - lib/metanorma/plugin/lutaml/lutaml_diagram_block_macro.rb
@@ -277,6 +279,7 @@ files:
 - lib/metanorma/plugin/lutaml/xmi_context_builder.rb
 - lib/metanorma/plugin/lutaml/xmi_package_filter.rb
 - lib/metanorma/plugin/lutaml/xmi_parse_cache.rb
+- lib/metanorma/plugin/lutaml/xmi_partial_load_registry.rb
 - lib/metanorma/plugin/lutaml/xmi_renderer.rb
 - lib/metanorma/plugin/lutaml/yaml2_text_preprocessor.rb
 - metanorma-plugin-lutaml.gemspec