leptris 1.9.33 → 1.9.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '01791cd8ecc5e2c7b472ac0333ae89ab338d6fa8872a55f724cafe708f866f2b'
-  data.tar.gz: 00d0115424789446417bce8b22d35a8b1f23883c17dd135f4a4e8d0d8ba19aa1
+  metadata.gz: f0445600493fa886524c162f270e55bccb3550d2c8b9463fe0a7ce305687a7ae
+  data.tar.gz: 878c7ecdcb828d80e3a25c0a05932113ce33030ccf163aea3896453fa0fc0bb8
 SHA512:
-  metadata.gz: a8d17f3f61f0317d9165a57e6ecf134cb22751c23f5fd39a669e9df7f6fa414e780a52ab55f7bd549c1ea10ae8cfa6d0fad6fe44f9b092ac581974d391cfa521
-  data.tar.gz: 2842e82675cba9aad3bc4bb859e1a85a78043b3b81b4eebf2d7c0afab4df3abacb01ddc56144d0bec96bd3473d5ad31bff9dc4beef31c2639282c9f7c55b84f1
+  metadata.gz: e9ab5ca5b140e729066e27a1afa5882639c7e18b6a10a94be06eccca5723ce0bf4547dd70940f367277548802caaca4a76d874148153fd55131ba5bb1d20aeb2
+  data.tar.gz: c8c62d53ff231fb9eedf7f8aa5a3d96774517bf26b347cc4b833ee8d149d9c80a37e014b1f4833bd2a7074eae4944d13819d89c292c8a0ccf0ead4b147101ff2

data/CHANGELOG.md

@@ -5,6 +5,37 @@ All notable changes to Leptris will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.9.34] - 2026-08-28
+
+### Changed
+
+- **SAX fully wins every shape** (leptris-ruby's parity-not-win,
+  upstream #594's framing): on the all-events handler profile the
+  binding previously LOST to Nokogiri by 14% (170 vs 149 ms per
+  1.9 MB parse) — the ffi gem's per-event callback trampoline
+  could not match a compiled C extension. Two things changed:
+  - The engine's C-side recorder cost no longer scales with
+    element count (~12 ms per 250k-event document now), so a bulk
+    transport can win.
+  - New `SAX::Recorder#dispatch(handler, kinds)` — the two-level
+    drain driving handler METHOD calls with the callback
+    transport's exact shapes (pairs arrays, one-arg arity
+    dispatch, UTF-8, PI normalization). `SAX::Parser` picks the
+    transport by override weight (characters 0.6, start/end 0.2;
+    ≥ 0.8 → bulk): one overridden hot kind stays on callbacks
+    (the engine skips C-side emission for unattached kinds:
+    text-only 23 ms vs 43 ms), two or more go bulk (start+end+chars
+    167 → **122 ms**, ahead of Nokogiri's 142 ms on the same
+    machine; head-to-head 123 vs 133). Handlers cannot tell which
+    transport served them — spec-pinned byte-identical call
+    sequences.
+  - Drain repairs that made it possible: kind-strip two-level
+    drain (no more 2.25M-Integer unpack array per single-feed
+    drain), `FFI::Struct.size` hoisted out of the loop (250k
+    layout lookups per parse), unused fields yield nil instead of
+    empty-String allocations, a characters fast path in the drain
+    loop.
+
 ## [1.9.33] - 2026-08-28
 
 ### Changed

data/lib/leptris/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Leptris
-  VERSION = "1.9.33"
+  VERSION = "1.9.34"
 end

data/lib/leptris/xml/sax/parser.rb

@@ -18,6 +18,7 @@ class Leptris::XML::SAX::Parser
   def document=(handler)
     @document = handler
     @handler_struct = nil
+    @dispatched_kinds = nil
   end
 
   # The handler struct (eleven FFI::Function callbacks + the C
@@ -40,16 +41,97 @@ class Leptris::XML::SAX::Parser
 
   def parse_memory(string)
     string = string.dup.force_encoding("UTF-8")
-    handler_struct = self.handler_struct
-    rc = Leptris::XML::FFI.leptris_sax_parse(
-      string, string.bytesize, handler_struct.pointer, nil)
-    if rc != 0
-      raise Leptris::XML::ParseError,
-        "leptris_sax_parse failed (rc=#{rc})"
+    if bulk_dispatch?
+      parse_memory_bulk(string)
+    else
+      handler_struct = self.handler_struct
+      rc = Leptris::XML::FFI.leptris_sax_parse(
+        string, string.bytesize, handler_struct.pointer, nil)
+      if rc != 0
+        raise Leptris::XML::ParseError,
+          "leptris_sax_parse failed (rc=#{rc})"
+      end
     end
     self
   end
 
+  # Bulk transport for many-override handlers: record the whole
+  # document C-side (~12ms per 250k events), then dispatch through
+  # the recorder's drain — 250k ffi callback trampolines cost
+  # ~150ms, so this wins outright for handlers that consume most of
+  # the event stream (and beats Nokogiri's C extension on the same
+  # shape). The delivered call shapes are IDENTICAL to the callback
+  # transport (pairs arrays, arity dispatch, UTF-8, PI-data
+  # normalization) — the transports are interchangeable to the
+  # handler.
+  #
+  # Transport choice, from the round-XXVI crossover measurements:
+  # ONE overridden hot kind pays less through the callbacks (the
+  # engine skips C-side emission for unattached kinds entirely:
+  # text-only 23ms vs 43ms), while two or more hot kinds win through
+  # the recorder (start+chars 138 -> 99ms, start+end+chars 167 ->
+  # 122ms vs Nokogiri's 142ms). The weights are document-shape
+  # priors — text-heavy documents with tagged structure.
+  HOT_KIND_WEIGHTS = {
+    characters: 0.6, start_element: 0.2, end_element: 0.2,
+  }.freeze
+  private_constant :HOT_KIND_WEIGHTS
+  DISPATCH_WEIGHT_THRESHOLD = 0.8
+  private_constant :DISPATCH_WEIGHT_THRESHOLD
+
+  def parse_memory_bulk(string)
+    recorder = Leptris::XML::SAX::Recorder.open
+    begin
+      rc = recorder.feed(string, final: true)
+      recorder.dispatch(@document, dispatched_kinds)
+      if rc != 0
+        raise Leptris::XML::ParseError,
+          "leptris_sax_parse failed (rc=#{rc})"
+      end
+    ensure
+      recorder.free
+    end
+    self
+  end
+
+  # The handler's overridden-kind map — the same decisions
+  # build_handler_struct makes, in dispatch form. Memoized with the
+  # handler struct; document= invalidates both.
+  def dispatched_kinds
+    @dispatched_kinds ||= begin
+      kinds = {}
+      {
+        start_document: :start_document,
+        end_document: :end_document,
+        start_element: :start_element,
+        end_element: :end_element,
+        characters: :characters,
+        comment: :comment,
+        cdata_block: :cdata,
+        processing_instruction: :pi,
+        start_prefix_mapping: :start_prefix,
+        end_prefix_mapping: :end_prefix,
+        error: :error,
+      }.each do |method_name, kind|
+        next unless overridden?(method_name)
+        kinds[kind] =
+          if method_name == :start_element &&
+              @document.method(:start_element).arity == 1
+            :one_arg
+          else
+            true
+          end
+      end
+      kinds
+    end
+  end
+
+  def bulk_dispatch?
+    dispatched_kinds.keys.sum do |kind|
+      HOT_KIND_WEIGHTS.fetch(kind, 0.0)
+    end >= DISPATCH_WEIGHT_THRESHOLD
+  end
+
   def parse_io(io)
     handler_struct = self.handler_struct
     parser_ptr = Leptris::XML::FFI.leptris_sax_parser_create(

data/lib/leptris/xml/sax/recorder.rb

@@ -88,6 +88,161 @@ class Leptris::XML::SAX::Recorder
     each_event(*Array(kinds), &block)
   end
 
+  # Which kinds carry each string field (the record layout leaves
+  # the other at offset 0 / length 0): slicing a zero-length run
+  # allocates a wasted empty String per event — 150k+ of them on a
+  # text-heavy document. Unused fields yield nil (the callback
+  # transport delivers no name for characters either).
+  HAS_NAME = {
+    start_element: true, end_element: true,
+    pi: true, start_prefix: true, end_prefix: true,
+  }.freeze
+  HAS_TEXT = {
+    characters: true, comment: true, cdata: true,
+    pi: true, start_prefix: true, error: true,
+  }.freeze
+
+  # Layout offsets (the struct stays the ABI's single source of
+  # truth) and per-count kind-strip templates: one byte per record,
+  # skipping the stride — the kind codes are 0..10, so the strip
+  # unpacks to cache-resident Fixnums, unlike the old whole-record
+  # unpack whose 2.25M-Integer array for a single-feed drain was
+  # most of the loop's allocation profile.
+  RECORD = Leptris::XML::FFI::SaxEventRecord
+  NAME_OFF = RECORD.offset_of(:name_off)
+  NAME_LEN = RECORD.offset_of(:name_len)
+  TEXT_OFF = RECORD.offset_of(:text_off)
+  TEXT_LEN = RECORD.offset_of(:text_len)
+  ATTRS_OFF = RECORD.offset_of(:attrs_off)
+  ATTR_COUNT = RECORD.offset_of(:attr_count)
+  LINE_OFF = RECORD.offset_of(:line)
+  COLUMN_OFF = RECORD.offset_of(:column)
+  RECORD_STRIDE = RECORD.size
+  SAX_EVENT_CHARACTERS_CODE = Leptris::XML::FFI::SAX_EVENT_CHARACTERS
+  NUL = "\0".freeze
+  KIND_TEMPLATE_FOR = ::Hash.new do |cache, count|
+    cache[count] = ("C x#{RECORD_STRIDE - 1}" * count)
+  end
+
+  # Dispatch drain: delivers the recorded events as handler method
+  # calls with the CALLBACK transport's exact shapes — pairs arrays
+  # (not hashes), one-argument start_element arity dispatch, UTF-8
+  # strings, PI-data whitespace normalization. +dispatched+ maps
+  # kind symbols to true for the kinds the handler overrides (the
+  # attach-only-overridden policy carried onto the bulk transport:
+  # other records cost one strip read). The C recorder costs ~12ms
+  # per 250k-event document while 250k ffi callback trampolines
+  # cost ~150ms — this is how a many-override handler beats the
+  # callback transport (and Nokogiri's C extension) outright.
+  def dispatch(handler, dispatched)
+    count_ptr = ::FFI::MemoryPointer.new(:size_t)
+    len_ptr = ::FFI::MemoryPointer.new(:size_t)
+    begin
+      records_ptr = Leptris::XML::FFI.leptris_sax_recorder_records(
+        @handle, count_ptr)
+      return self if records_ptr.null? || count_ptr.read_uint64.zero?
+      arena_ptr = Leptris::XML::FFI.leptris_sax_recorder_arena(
+        @handle, len_ptr)
+      arena = arena_ptr.read_bytes(len_ptr.read_uint64)
+      count = count_ptr.read_uint64
+      records = records_ptr.read_bytes(count * RECORD_STRIDE)
+      kinds = records.unpack(KIND_TEMPLATE_FOR[count])
+
+      one_arg_start = dispatched[:start_element] == :one_arg
+      i = 0
+      while i < count
+        code = kinds[i]
+        kind = KIND_BY_CODE[code]
+        if kind && dispatched[kind]
+          base = i * RECORD_STRIDE
+          case kind
+          when :characters
+            handler.characters(slice(arena,
+              records_ptr.get_uint32(base + TEXT_OFF),
+              records_ptr.get_uint32(base + TEXT_LEN)))
+          when :start_element
+            name = slice(arena,
+              records_ptr.get_uint32(base + NAME_OFF),
+              records_ptr.get_uint32(base + NAME_LEN))
+            if one_arg_start
+              handler.start_element(name)
+            else
+              handler.start_element(name, attrs_pairs(arena,
+                records_ptr.get_uint32(base + ATTRS_OFF),
+                records_ptr.get_uint32(base + ATTR_COUNT)))
+            end
+          when :end_element
+            handler.end_element(slice(arena,
+              records_ptr.get_uint32(base + NAME_OFF),
+              records_ptr.get_uint32(base + NAME_LEN)))
+          when :comment
+            handler.comment(slice(arena,
+              records_ptr.get_uint32(base + TEXT_OFF),
+              records_ptr.get_uint32(base + TEXT_LEN)))
+          when :cdata
+            handler.cdata_block(slice(arena,
+              records_ptr.get_uint32(base + TEXT_OFF),
+              records_ptr.get_uint32(base + TEXT_LEN)))
+          when :pi
+            handler.processing_instruction(slice(arena,
+              records_ptr.get_uint32(base + NAME_OFF),
+              records_ptr.get_uint32(base + NAME_LEN)),
+              Leptris::XML::FFI.read_pi_data(slice(arena,
+                records_ptr.get_uint32(base + TEXT_OFF),
+                records_ptr.get_uint32(base + TEXT_LEN))))
+          when :start_prefix
+            handler.start_prefix_mapping(slice(arena,
+              records_ptr.get_uint32(base + NAME_OFF),
+              records_ptr.get_uint32(base + NAME_LEN)),
+              slice(arena,
+                records_ptr.get_uint32(base + TEXT_OFF),
+                records_ptr.get_uint32(base + TEXT_LEN)))
+          when :end_prefix
+            handler.end_prefix_mapping(slice(arena,
+              records_ptr.get_uint32(base + NAME_OFF),
+              records_ptr.get_uint32(base + NAME_LEN)))
+          when :error
+            handler.error(slice(arena,
+              records_ptr.get_uint32(base + TEXT_OFF),
+              records_ptr.get_uint32(base + TEXT_LEN)),
+              records_ptr.get_uint32(base + LINE_OFF),
+              records_ptr.get_uint32(base + COLUMN_OFF))
+          when :start_document
+            handler.start_document
+          when :end_document
+            handler.end_document
+          end
+        end
+        i += 1
+      end
+    ensure
+      count_ptr.free
+      len_ptr.free
+    end
+    self
+  end
+
+  # attrs as [name, value] pairs in source order — the callback
+  # transport's shape (walk_attr_array).
+  def attrs_pairs(arena, off, count)
+    return [] if count.zero?
+    pairs = []
+    pos = off
+    nul = NUL
+    utf8 = Encoding::UTF_8
+    while count > 0
+      name_end = arena.index(nul, pos)
+      name = arena.byteslice(pos, name_end - pos).force_encoding(utf8)
+      pos = name_end + 1
+      value_end = arena.index(nul, pos)
+      pairs << [name,
+        arena.byteslice(pos, value_end - pos).force_encoding(utf8)]
+      pos = value_end + 1
+      count -= 1
+    end
+    pairs
+  end
+
   # Drains the current chunk's records: bulk-reads the record array
   # and the packed arena (two FFI calls), then slices every string
   # from the arena in Ruby — no per-string FFI. With +kinds+, records
@@ -108,27 +263,48 @@ class Leptris::XML::SAX::Recorder
       arena = arena_ptr.read_bytes(len_ptr.read_uint64)
       count = count_ptr.read_uint64
 
-      # The whole record block in ONE read: read_bytes + String#unpack
-      # (C-fast) — two FFI calls per chunk total (records + arena),
-      # the true bulk discipline. Template per record: kind byte,
-      # 7 pad, then the eight uint32 fields (little-endian hosts —
-      # the layout the header freezes for FFI mirrors).
-      stride = Leptris::XML::FFI::SaxEventRecord.size
-      fields = records_ptr.read_bytes(count * stride)
-        .unpack("C x7 V8" * count)
+      # Two-level drain: ONE bulk read + kind-strip unpack decides
+      # which records to touch, then each yielded event's fields are
+      # read individually (get_uint32 at layout offsets) — an
+      # unwanted record costs one strip read, an unused FIELD costs
+      # nothing, and no per-chunk field array is ever built.
+      records = records_ptr.read_bytes(count * RECORD_STRIDE)
+      kinds = records.unpack(KIND_TEMPLATE_FOR[count])
 
+      has_name = HAS_NAME
+      has_text = HAS_TEXT
+      drain = wanted.nil?
       i = 0
       while i < count
-        kind = KIND_BY_CODE[fields[i * 9]]
-        if kind && (wanted.nil? || wanted[kind])
-          base = i * 9 + 1
-          text = slice(arena, fields[base + 2], fields[base + 3])
-          text = Leptris::XML::FFI.read_pi_data(text) if kind == :pi
-          yield kind,
-                slice(arena, fields[base], fields[base + 1]),
-                text,
-                attrs_from(arena, fields[base + 4], fields[base + 5]),
-                fields[base + 6], fields[base + 7]
+        code = kinds[i]
+        base = i * RECORD_STRIDE
+        if code == SAX_EVENT_CHARACTERS_CODE && drain
+          # The dominant branch (text-heavy documents): no name, no
+          # attrs, no per-kind gate lookups — kind, text, position.
+          yield :characters, nil,
+                slice(arena, records_ptr.get_uint32(base + TEXT_OFF),
+                             records_ptr.get_uint32(base + TEXT_LEN)),
+                nil,
+                records_ptr.get_uint32(base + LINE_OFF),
+                records_ptr.get_uint32(base + COLUMN_OFF)
+        else
+          kind = KIND_BY_CODE[code]
+          if kind && (drain || wanted[kind])
+            name = if has_name[kind]
+                     slice(arena, records_ptr.get_uint32(base + NAME_OFF),
+                                  records_ptr.get_uint32(base + NAME_LEN))
+                   end
+            text = if has_text[kind]
+                     slice(arena, records_ptr.get_uint32(base + TEXT_OFF),
+                                  records_ptr.get_uint32(base + TEXT_LEN))
+                   end
+            text = Leptris::XML::FFI.read_pi_data(text) if kind == :pi
+            yield kind, name, text,
+                  attrs_from(arena, records_ptr.get_uint32(base + ATTRS_OFF),
+                                    records_ptr.get_uint32(base + ATTR_COUNT)),
+                  records_ptr.get_uint32(base + LINE_OFF),
+                  records_ptr.get_uint32(base + COLUMN_OFF)
+          end
         end
         i += 1
       end
@@ -153,9 +329,9 @@ class Leptris::XML::SAX::Recorder
     kinds.each_with_object({}) { |kind, set| set[kind] = true }
   end
 
-  # Zero-length slices are empty strings, not nil — the default
-  # namespace prefix is legitimately "" (START_PREFIX: name may be
-  # ""), matching the callback transport's delivery.
+  # The default-namespace prefix is legitimately "" (START_PREFIX:
+  # name may be zero-length) — zero-length slices stay "" where the
+  # field is carried (unused fields never reach here).
   def slice(arena, off, len)
     arena.byteslice(off, len).force_encoding(Encoding::UTF_8)
   end
@@ -166,15 +342,17 @@ class Leptris::XML::SAX::Recorder
     return nil if count.zero?
     hash = {}
     pos = off
-    count.times do
-      name_end = arena.index("\0", pos)
-      name = arena.byteslice(pos, name_end - pos)
+    nul = NUL
+    utf8 = Encoding::UTF_8
+    while count > 0
+      name_end = arena.index(nul, pos)
+      name = arena.byteslice(pos, name_end - pos).force_encoding(utf8)
       pos = name_end + 1
-      value_end = arena.index("\0", pos)
-      value = arena.byteslice(pos, value_end - pos)
+      value_end = arena.index(nul, pos)
+      hash[name] =
+        arena.byteslice(pos, value_end - pos).force_encoding(utf8)
       pos = value_end + 1
-      hash[name.force_encoding(Encoding::UTF_8)] =
-        value.force_encoding(Encoding::UTF_8)
+      count -= 1
     end
     hash
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: leptris
 version: !ruby/object:Gem::Version
-  version: 1.9.33
+  version: 1.9.34
 platform: ruby
 authors:
 - Ribose Inc.