metanorma-utils 2.0.7 → 2.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76df06597b667091ccab17e9c0229f4304769d1c21b762bac6f55494a8314c16
-  data.tar.gz: 75e494060f32426997f7637a937148866b148d34699d65e18d53309b4070ed7b
+  metadata.gz: 4cb1c9e13d72f7f7310918941cd4df7243968915307e4ec830160237b715ea54
+  data.tar.gz: 9219b32ec8d1672f82d5c3e7d88f1903d39e4e19ddb481f8c7748c55810779c3
 SHA512:
-  metadata.gz: f6ccbe14dc7d8bda84b6b9f8ac1d5c5ea94a149d3f4da74222bb3395e108d42fe60960a922e9db595401ccf48179f9a1f5f6a93424c29cbc1ce9365b3c5437ad
-  data.tar.gz: b77a8e2e913de8c9d2e56178d73ced28ed2ed90857eca8cf61ba0347b5a936a5f78386c9dd08e18fc40a9e297cf399617d7fc93d7923527c741763e179ba0940
+  metadata.gz: f7821e4c4b22b02182b5a66e561fcccfd8a0dc2cfdb304f347eab986d5c5a444565380affcb80a11c6e96a1fcf1cedf255b8c82d8eaccd3dab2937664f66622b
+  data.tar.gz: f1a9df51ff4cf0b8c3376407b63f45ef4766845bd3c6659e4f479956df60bfeba1307e86ae91ebc7a033c45a88d25199e0ce9dc79f5b636d9f922a9466eec7cd

data/.rubocop.yml

@@ -1,10 +1,31 @@
 # Auto-generated by Cimas: Do not edit it manually!
 # See https://github.com/metanorma/cimas
 inherit_from:
-  - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+  - https://raw.githubusercontent.com/riboseinc/oss-guides/main/ci/rubocop.yml
+  # .rubocop_todo.yml MUST be the last entry. inherit_from is last-wins:
+  # if listed before oss-guides, the shared config's stricter Metrics/
+  # (MethodLength, BlockLength, etc.) rules override the todo's per-file
+  # grandfathering, and the todo becomes inert on those cops. Empirically
+  # verified on suma 2026-07-06: with todo listed first, 34 Metrics/* offenses
+  # remained; moved last, cleared. Every gem in the metanorma-org fleet
+  # already ships a `.rubocop_todo.yml` on its live tree (audited 2026-07-06,
+  # 54/54 have it), so this reference is safe to emit unconditionally.
+  - .rubocop_todo.yml
+
+# Rubocop plugins enabled centrally so every metanorma-org gem picks them up
+# on cimas sync — best practice belongs at the shared-template layer, not
+# per-repo. Per ronaldtse feedback on metanorma/ci#332.
+plugins:
+  - rubocop-rspec
+  - rubocop-performance
+  - rubocop-rake
 
 # local repo-specific modifications
 # ...
 
 AllCops:
-  TargetRubyVersion: 2.5
+  # 3.3 matches the org-wide minimum being pushed via #274 (Raise minimum
+  # Ruby version to 3.3 due to EOL of 3.2 on 2026-03-31). Was 3.4 before
+  # this commit — 3.4 was above the org's stated minimum and would have
+  # applied Rubocop rules that fail-close on gems still targeting 3.3.
+  TargetRubyVersion: 3.3

data/lib/metanorma-utils.rb

@@ -7,3 +7,5 @@ require_relative "utils/log"
 require_relative "utils/xml"
 require_relative "utils/anchor"
 require_relative "utils/anchor_ranges"
+require_relative "utils/gc_budget"
+require_relative "utils/asciidoctor_table_buffer"

data/lib/utils/anchor.rb

@@ -105,7 +105,7 @@ module Metanorma
       def to_ncname_prep(name, asciionly)
         name = name&.to_s
         name.nil? and name = ""
-        asciionly and name = HTMLEntities.new.encode(name,
+        asciionly and name = HTML_ENTITIES.encode(name,
                                                      :basic, :hexadecimal)
         [name, name.nil? || name.empty? || name.match?(SAFE_NCNAME_REGEXP)]
       end

data/lib/utils/asciidoctor_table_buffer.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+require "asciidoctor"
+
+# Asciidoctor 2.0.x accumulates a multi-line table cell by rebuilding the
+# whole cell buffer String on every appended line
+# (`parser_ctx.buffer = %(#{parser_ctx.buffer}#{line}#{LF})` in
+# Asciidoctor::Parser.parse_table), so building a cell that spans N lines
+# is quadratic in the bytes of the cell. Generated tables with cells that
+# span tens of thousands of lines allocate ~10 GB of transient Strings
+# and exceed modest memory caps before the table finishes parsing.
+#
+# The patch below carries the fix inside metanorma-utils: it prepends a
+# corrected Asciidoctor::Parser.parse_table whose appends go through
+# ParserContext#append_to_buffer, and prepends ParserContext methods that
+# accumulate the cell as an Array of segments joined once at close (and
+# whenever the CSV quote check needs the whole record). No String owned
+# by the buffer is ever mutated, so no frozen-string workaround is
+# needed, and the per-line buffer-sized String churn the GC chases is
+# gone.
+#
+# This is also the shape the asciidoctor maintainers have stated they
+# would accept upstream: in-place string mutation is barred there by
+# frozen-string literals and Opal compatibility, and their stated
+# solution is "an array that is later concatenated" (the comment thread
+# of asciidoctor PR #4879). An upstream fix of either stated shape stands
+# the prepend down (see the disarm check at the end of this file).
+module Asciidoctor
+  # Replaces the buffer-mutating ParserContext methods; the two stock
+  # readers of the buffer (the CSV quote check and close_cell) drain the
+  # pending segments into the materialized buffer and delegate upward.
+  module TableCellBufferParserContextPatch
+    # Appends the String to the buffer of the currently open cell as a
+    # pending segment; amortized O(1) per accumulated line.
+    def append_to_buffer(str)
+      (@buffer_segments ||= []) << str
+      nil
+    end
+
+    def skip_past_delimiter(pre)
+      (@buffer_segments ||= []) << pre << @delimiter
+      nil
+    end
+
+    def skip_past_escaped_delimiter(pre)
+      (@buffer_segments ||= []) << pre.chop << @delimiter
+      nil
+    end
+
+    def buffer_has_unclosed_quotes?(append = nil, q = '"')
+      drain_buffer_segments
+      super
+    end
+
+    def close_cell(eol = false)
+      drain_buffer_segments
+      super
+    end
+
+    private
+
+    # Materializes the pending segments onto the buffer String the stock
+    # methods read. Cost is proportional to the pending segments only;
+    # close_cell drains once per cell, and the quote check drains at most
+    # once per check - the same order as the stock check's own copy of
+    # the full record ((@buffer + append).strip).
+    def drain_buffer_segments
+      (@buffer_segments ||= []).empty? and return
+      @buffer = @buffer + @buffer_segments.join
+      @buffer_segments.clear
+      nil
+    end
+  end
+
+  module TableCellBufferPatch
+  def parse_table(table_reader, parent, attributes)
+    table = Table.new(parent, attributes)
+
+    if (attributes.key? 'cols') && !(colspecs = parse_colspecs attributes['cols']).empty?
+      table.create_columns colspecs
+      explicit_colspecs = true
+    end
+
+    skipped = table_reader.skip_blank_lines || 0
+    if attributes['header-option']
+      table.has_header_option = true
+    elsif skipped == 0 && !attributes['noheader-option']
+      # NOTE: assume table has header until we know otherwise; if it doesn't (nil), cells in first row get reprocessed
+      table.has_header_option = :implicit
+      implicit_header = true
+    end
+    parser_ctx = Table::ParserContext.new table_reader, table, attributes
+    format, loop_idx, implicit_header_boundary = parser_ctx.format, -1, nil
+
+    while (line = table_reader.read_line)
+      if (beyond_first = (loop_idx += 1) > 0) && line.empty?
+        line = nil
+        implicit_header_boundary += 1 if implicit_header_boundary
+      elsif format == 'psv'
+        if parser_ctx.starts_with_delimiter? line
+          line = line.slice 1, line.length
+          # push empty cell spec if cell boundary appears at start of line
+          parser_ctx.close_open_cell
+          implicit_header_boundary = nil if implicit_header_boundary
+        else
+          next_cellspec, line = parse_cellspec line, :start, parser_ctx.delimiter
+          # if cellspec is not nil, we're at a cell boundary
+          if next_cellspec
+            parser_ctx.close_open_cell next_cellspec
+            implicit_header_boundary = nil if implicit_header_boundary
+          # otherwise, the cell continues from previous line
+          elsif implicit_header_boundary && implicit_header_boundary == loop_idx
+            table.has_header_option = implicit_header = implicit_header_boundary = nil
+          end
+        end
+      end
+
+      unless beyond_first
+        table_reader.mark
+        # NOTE implicit header is offset by at least one blank line; implicit_header_boundary tracks size of gap
+        if implicit_header
+          if table_reader.has_more_lines? && table_reader.peek_line.empty?
+            implicit_header_boundary = 1
+          else
+            table.has_header_option = implicit_header = nil
+          end
+        end
+      end
+
+      # this loop is used for flow control; internal logic controls how many times it executes
+      while true
+        if line && (m = parser_ctx.match_delimiter line)
+          pre_match, post_match = m.pre_match, m.post_match
+          case format
+          when 'csv'
+            if parser_ctx.buffer_has_unclosed_quotes? pre_match
+              parser_ctx.skip_past_delimiter pre_match
+              break if (line = post_match).empty?
+              redo
+            end
+            parser_ctx.append_to_buffer pre_match
+          when 'dsv'
+            if pre_match.end_with? '\\'
+              parser_ctx.skip_past_escaped_delimiter pre_match
+              if (line = post_match).empty?
+                parser_ctx.append_to_buffer LF
+                parser_ctx.keep_cell_open
+                break
+              end
+              redo
+            end
+            parser_ctx.append_to_buffer pre_match
+          else # psv
+            if pre_match.end_with? '\\'
+              parser_ctx.skip_past_escaped_delimiter pre_match
+              if (line = post_match).empty?
+                parser_ctx.append_to_buffer LF
+                parser_ctx.keep_cell_open
+                break
+              end
+              redo
+            end
+            next_cellspec, cell_text = parse_cellspec pre_match
+            parser_ctx.push_cellspec next_cellspec
+            parser_ctx.append_to_buffer cell_text
+          end
+          # don't break if empty to preserve empty cell found at end of line (see issue #1106)
+          line = nil if (line = post_match).empty?
+          parser_ctx.close_cell
+        else
+          # no other delimiters to see here; suck up this line into the buffer and move on
+          parser_ctx.append_to_buffer %(#{line}#{LF})
+          case format
+          when 'csv'
+            if parser_ctx.buffer_has_unclosed_quotes?
+              table.has_header_option = implicit_header = implicit_header_boundary = nil if implicit_header_boundary && loop_idx == 0
+              parser_ctx.keep_cell_open
+            else
+              parser_ctx.close_cell true
+            end
+          when 'dsv'
+            parser_ctx.close_cell true
+          else # psv
+            parser_ctx.keep_cell_open
+          end
+          break
+        end
+      end
+
+      # NOTE cell may already be closed if table format is csv or dsv
+      if parser_ctx.cell_open?
+        parser_ctx.close_cell true unless table_reader.has_more_lines?
+      else
+        table_reader.skip_blank_lines || break
+      end
+    end
+
+    parser_ctx.close_table
+    table.assign_column_widths unless (table.attributes['colcount'] ||= table.columns.size) == 0 || explicit_colspecs
+    table.has_header_option = true if implicit_header
+    table.partition_header_footer attributes
+
+    table
+  end
+  end
+
+  class Table
+    class ParserContext
+      prepend TableCellBufferParserContextPatch
+    end
+  end
+end
+
+module Metanorma
+  module Utils
+    module AsciidoctorTableBuffer
+      # Resolved once, before the prepend: once our parse_table is in
+      # place, Parser.method(:parse_table) resolves to the prepended copy
+      # and its source_location points at this file, not asciidoctor's.
+      PARSER_FILE = ::Asciidoctor::Parser
+        .method(:parse_table).source_location.to_a.first
+
+      class << self
+        def apply!
+          return false unless defined?(::Asciidoctor::VERSION)
+          return true if applied?
+
+          parser_file = PARSER_FILE
+          return false unless parser_file && File.file?(parser_file)
+          # Stand down once asciidoctor fixes the quadratic rebuild
+          # itself, in either shape: appending in place
+          # (append_to_buffer), or accumulating lines in an Array
+          # joined at close - the shape the asciidoctor maintainers
+          # have stated is acceptable upstream (string mutation is
+          # barred there by frozen-string literals and Opal
+          # compatibility; see the comment thread of asciidoctor PR
+          # #4879). Detecting both matters: this patch prepends a whole
+          # copy of parse_table, so shadowing an upstream fix with a
+          # stale copy would silently revert their behavior.
+          parser_src = File.read(parser_file)
+          return false if parser_src.include?("append_to_buffer")
+          return false unless parser_src.include?('parser_ctx.buffer = %(#{parser_ctx.buffer')
+
+          ::Asciidoctor::Parser.singleton_class
+            .prepend(::Asciidoctor::TableCellBufferPatch)
+          ::Asciidoctor::Table::ParserContext
+            .prepend(::Asciidoctor::TableCellBufferParserContextPatch)
+          @applied = true
+        end
+
+        def applied?
+          @applied ||= false
+        end
+      end
+    end
+  end
+end
+
+Metanorma::Utils::AsciidoctorTableBuffer.apply!

data/lib/utils/gc_budget.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Utils
+    # Large metanorma compiles churn transients of hundreds of MB per
+    # phase (macro rendering, XML cleanup, whole-document assembly) on
+    # top of a multi-GB live set. The churn is malloc-side
+    # (libxml/moxml/liquid/regexp), so the GC's heap-growth heuristics
+    # fire too late under memory caps and the process dies thrashing.
+    # Call gc_when_bloated! at phase boundaries: collect whenever
+    # uncollected malloc has exceeded the budget.
+    module GcBudget
+      BUDGET_BYTES = 512 * (1 << 20)
+
+      class << self
+        def gc_when_bloated!
+          GC.start if GC.stat(:malloc_increase_bytes) > BUDGET_BYTES
+        end
+      end
+    end
+  end
+end

data/lib/utils/main.rb

@@ -9,6 +9,12 @@ require_relative "cjk"
 
 module Metanorma
   module Utils
+    # One shared decoder: instantiating HTMLEntities builds a full entity
+    # table per instance, and these helpers run per attribute value / per
+    # text node in document-scale pipelines — hundreds of thousands of
+    # live decoders were observed retained in a single metanorma compile.
+    HTML_ENTITIES = HTMLEntities.new
+
     class << self
       # , " => ," : CSV definition does not deal with space followed by quote
       # at start of field
@@ -38,7 +44,7 @@ module Metanorma
 
       # TODO needs internationalisation of quote
       def smartformat(text)
-        ret = HTMLEntities.new.decode(
+        ret = HTML_ENTITIES.decode(
           text.gsub(/ --? /, "&#8201;&#8212;&#8201;")
           .gsub("--", "&#8212;"),
         )
@@ -47,7 +53,7 @@ module Metanorma
         ret = ret.smart_format
         ret = ret.gsub(%r{(#{CJK})\u200a}o, "\\1")
           .gsub(%r{\u200a(#{CJK})}o, "\\1")
-        HTMLEntities.new.encode(ret, :basic)
+        HTML_ENTITIES.encode(ret, :basic)
       end
 
       def endash_date(elem)

data/lib/utils/version.rb

@@ -1,5 +1,5 @@
 module Metanorma
   module Utils
-    VERSION = "2.0.7".freeze
+    VERSION = "2.0.8".freeze
   end
 end

data/lib/utils/xml.rb

@@ -30,17 +30,29 @@ module Metanorma
     HERE
 
     class << self
+      # The parsed NOKOHEAD scaffold, shared by every noko fragment.
+      def noko_template
+        @noko_template ||= ::Nokogiri::XML.parse(NOKOHEAD)
+      end
+
       def attr_code(attributes)
         attributes.compact.transform_values do |v|
-          v.is_a?(String) ? HTMLEntities.new.decode(v) : v
+          v.is_a?(String) ? HTML_ENTITIES.decode(v) : v
         end
       end
 
       # block for processing XML document fragments as XHTML,
       # to allow for HTMLentities
       # Unescape special chars used in Asciidoctor substitution processing
+      # The template document is parsed once per process: noko runs for
+      # every inline element and block a converter emits (a metanorma
+      # plateau compile measured 73k calls), and re-parsing NOKOHEAD
+      # minted a fresh HTML-skeleton Document per call — 75k retained
+      # documents with their node caches, a top memory consumer of
+      # large compiles. Fragments off the shared document are
+      # independent trees; conversion is single-threaded.
       def noko(_script = "Latn", &block)
-        fragment = ::Nokogiri::XML.parse(NOKOHEAD).fragment("")
+        fragment = noko_template.fragment("")
         ::Nokogiri::XML::Builder.with fragment, &block
         fragment
           .to_xml(encoding: "UTF-8", indent: 0,
@@ -74,7 +86,7 @@ module Metanorma
       end
 
       def noko_html(&block)
-        doc = ::Nokogiri::XML.parse(NOKOHEAD)
+        doc = noko_template.dup
         fragment = doc.fragment("")
         ::Nokogiri::XML::Builder.with fragment, &block
         fragment.to_xml(encoding: "UTF-8", indent: 0,
@@ -97,7 +109,7 @@ module Metanorma
       end
 
       def numeric_escapes(xml)
-        c = HTMLEntities.new
+        c = HTML_ENTITIES
         xml.split(/(&[^ \r\n\t#&;]+;)/).map do |t|
           if /^(&[^ \t\r\n#;]+;)/.match?(t)
             c.encode(c.decode(t), :hexadecimal)

data/metanorma-utils.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features|bin|.github)/}) \
     || f.match(%r{Rakefile|bin/rspec})
   end
-  spec.required_ruby_version = Gem::Requirement.new(">= 3.1.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.0")
 
   spec.add_dependency "asciidoctor", ">= 2"
   spec.add_dependency "concurrent-ruby"
@@ -47,6 +47,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "simplecov", "~> 0.15"
   spec.add_development_dependency "timecop", "~> 0.9"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "canon", "= 0.2.3"
+  spec.add_development_dependency "canon"
   # spec.metadata["rubygems_mfa_required"] = "true"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma-utils
 version: !ruby/object:Gem::Version
-  version: 2.0.7
+  version: 2.0.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-05-25 00:00:00.000000000 Z
+date: 2026-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -294,16 +294,16 @@ dependencies:
   name: canon
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.3
+        version: '0'
 description: 'metanorma-utils provides utilities for the Metanorma stack
 
   '
@@ -314,7 +314,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".hound.yml"
 - ".rubocop.yml"
 - Gemfile
 - LICENSE
@@ -323,7 +322,9 @@ files:
 - lib/sterile/sterile.rb
 - lib/utils/anchor.rb
 - lib/utils/anchor_ranges.rb
+- lib/utils/asciidoctor_table_buffer.rb
 - lib/utils/cjk.rb
+- lib/utils/gc_budget.rb
 - lib/utils/hash_transform_keys.rb
 - lib/utils/image.rb
 - lib/utils/linestatus.rb
@@ -346,7 +347,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/.hound.yml

@@ -1,5 +0,0 @@
-# Auto-generated by Cimas: Do not edit it manually!
-# See https://github.com/metanorma/cimas
-ruby:
-  enabled: true
-  config_file: .rubocop.yml