parsanol 1.3.13 → 1.3.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78de43242c21f1c37c83a6b432227683162687a6d9e46fc2a86f31bc451e576e
-  data.tar.gz: 49a1bacad7a5006a471c0f8c194dd7931813869132a762c59fd928b1fde7cc40
+  metadata.gz: c4577d99080eb74c30e3de09fe01d342818d6776e8ee7e852b0de178c131469c
+  data.tar.gz: 332412b5d27238d167bb78e68d1bc9e378139bc9bc3377b304bfc201f91d7b51
 SHA512:
-  metadata.gz: e4a02c77e7f95bb2ec1b9ac87393f77842d82c83c1c0f52f6aa53565908fd6328c2b93c6f3ebe80cad9605b4deab9d628faed737c5bfb656133ffa2b331b150c
-  data.tar.gz: 3357e9dbab15be8e2020dac99429b0aa92c474b8a63e2e4fd21d86c825c87550bfb649105965d62ba17ea7abd6570ab357999984fe7022f38e8fd1561cf4dd2c
+  metadata.gz: 71f3e40a0eafe7a15d543eab66911393cee50f478fda9ab65c8986dd54b3ceba116f1f61cea2b7801c2a2d061e7c6d33e505fc4111b69fa2f04ef2eb1685528a
+  data.tar.gz: 5ce7080b01d79c2f18438115eaa29a96afce668a3ebdd6849f724b72b9d9aae90263c3db2aec9fcec478ff840532ff9c9e52c93be61e5801f5c961c5cbd05452

data/Cargo.lock

@@ -253,7 +253,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 [[package]]
 name = "parsanol"
 version = "0.5.1"
-source = "git+https://github.com/parsanol/parsanol-rs?branch=main#1a141f1a3471ba6ce83a768354a917d4ab83d172"
+source = "git+https://github.com/parsanol/parsanol-rs?branch=main#f6782c030b42aaa722357cd107fd56ed9853c4ea"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -270,7 +270,7 @@ dependencies = [
 [[package]]
 name = "parsanol-derive"
 version = "0.5.1"
-source = "git+https://github.com/parsanol/parsanol-rs?branch=main#1a141f1a3471ba6ce83a768354a917d4ab83d172"
+source = "git+https://github.com/parsanol/parsanol-rs?branch=main#f6782c030b42aaa722357cd107fd56ed9853c4ea"
 dependencies = [
  "proc-macro2",
  "quote",

data/lib/parsanol/atoms/can_flatten.rb

@@ -65,18 +65,25 @@ module Parsanol
         raise "BUG: Unknown tag #{tag.inspect}."
       end
 
-      # Lisp style fold left where the first element builds the basis for
-      # an inject. Optimized with early return and reduced method calls.
+      # Lisp style fold left where the first non-nil element builds the
+      # basis for an inject. Nil entries are skipped in a single pass so
+      # callers do not need an intermediate #compact.
       #
       def foldl(list)
         len = list.size
-        return "" if len.zero?
-        return list[0] if len == 1 # Fast path for single element
+        i = 0
+        while i < len && list[i].nil?
+          i += 1
+        end
+        return "" if i >= len
+
+        result = list[i]
+        i += 1
+        return result if i >= len # Fast path for single non-nil element
 
-        result = list[0]
-        i = 1
         while i < len
-          result = yield(result, list[i])
+          e = list[i]
+          result = yield(result, e) unless e.nil?
           i += 1
         end
         result
@@ -87,7 +94,7 @@ module Parsanol
       # @api private
       #
       def flatten_sequence(list)
-        foldl(list.compact) do |r, e| # and then merge flat elements
+        foldl(list) do |r, e|
           merge_fold(r, e)
         end
       end
@@ -161,22 +168,65 @@ module Parsanol
         if has_hash
           # If keyed subtrees are in the array, we'll want to discard all
           # strings inbetween. To keep them, name them.
-          return list.select { |e| e.instance_of?(Hash) }
+          hashes = []
+          i = 0
+          while i < len
+            e = list[i]
+            hashes << e if e.instance_of?(Hash)
+            i += 1
+          end
+          return hashes
         end
 
         if has_array
           # If any arrays are nested in this array, flatten all arrays to this
           # level.
-          return list
-              .select { |e| e.instance_of?(Array) }
-              .flatten(1)
+          flat = []
+          i = 0
+          while i < len
+            e = list[i]
+            if e.instance_of?(Array)
+              flat.concat(e)
+            end
+            i += 1
+          end
+          return flat
         end
 
         # Consistent handling of empty lists, when we act on a named result
         return [] if named && list.empty?
 
-        # If there are only strings, concatenate them and return that.
-        foldl(list.compact) { |s, e| s + e }
+        # All-string (or Slice) repetition: join once instead of O(n)
+        # intermediate Slice allocations via repeated #+.
+        join_string_list(list)
+      end
+
+      # Join a list of string-like values into a single Slice (or String),
+      # preserving the first Slice's offset and input. Nils are skipped.
+      def join_string_list(list)
+        first_slice = nil
+        content = nil
+        i = 0
+        len = list.size
+        while i < len
+          e = list[i]
+          unless e.nil?
+            if first_slice.nil? && e.instance_of?(Parsanol::Slice)
+              first_slice = e
+            end
+            piece = e.instance_of?(Parsanol::Slice) ? e.content : e.to_s
+            if content.nil?
+              content = +piece
+            else
+              content << piece
+            end
+          end
+          i += 1
+        end
+        return "" if content.nil?
+        return first_slice.class.new(first_slice.offset, content, first_slice.input) if first_slice
+
+        content
       end
 
       # That annoying warning 'Duplicate subtrees while merging result' comes

data/lib/parsanol/atoms/repetition.rb

@@ -171,16 +171,15 @@ module Parsanol
         context.err_at(self, source, @min_error, source.bytepos, [v3])
       end
 
-      # General repetition with buffer pooling
+      # General repetition: builds the tagged result Array in place so
+      # finalize_result can flatten without an intermediate Buffer/LazyResult
+      # materialization.
       def try_general(source, context, consume_all)
         start_pos = source.bytepos
         occurrence = 0
-
-        # Estimate buffer size
-        estimate = [@max || 10, 10].min
-        buffer = context.acquire_buffer(size: estimate + 1)
-        buffer.push(@result_tag)
-
+        # Pre-size for the tag + a modest run; Array growth is amortized.
+        result = Array.new([@max || 8, 8].min + 1)
+        result[0] = @result_tag
         last_error = nil
 
         loop do
@@ -190,14 +189,13 @@ module Parsanol
           break unless success
 
           occurrence += 1
-          buffer.push(value)
+          result[occurrence] = value
 
           break if @max && occurrence >= @max
         end
 
         # Check minimum bound
         if occurrence < @min
-          context.release_buffer(buffer)
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
                                 [last_error])
@@ -205,11 +203,12 @@ module Parsanol
 
         # Check complete consumption
         if consume_all && source.chars_left.positive?
-          context.release_buffer(buffer)
           return context.err(self, source, @extra_error, [last_error])
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        # Trim to the actual filled length (tag + occurrence matches).
+        result.pop(result.size - occurrence - 1) if result.size > occurrence + 1
+        ok(result)
       end
 
       # Tree memoization for GPEG-style caching
@@ -227,23 +226,21 @@ module Parsanol
 
         # Parse and cache
         occurrence = 0
-        estimate = [@max || 10, 10].min
-        buffer = context.acquire_buffer(size: estimate + 1)
-        buffer.push(@result_tag)
+        result = Array.new([@max || 8, 8].min + 1)
+        result[0] = @result_tag
 
         positions = context.acquire_array
         positions << start_pos
         last_error = nil
 
         loop do
-          source.bytepos
           success, value = @parslet.apply(source, context, false)
           last_error = value
 
           break unless success
 
           occurrence += 1
-          buffer.push(value)
+          result[occurrence] = value
           positions << source.bytepos
 
           break if @max && occurrence >= @max
@@ -252,13 +249,13 @@ module Parsanol
         # Cache successful prefix
         if occurrence.positive?
           end_pos = positions[occurrence]
-          context.store_tree_memo(cache_key, start_pos, buffer.to_a[1..],
-                                  end_pos)
+          context.store_tree_memo(cache_key, start_pos,
+                                  result[1, occurrence], end_pos)
         end
+        context.release_array(positions) if context.respond_to?(:release_array)
 
         # Check minimum
         if occurrence < @min
-          context.release_buffer(buffer)
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
                                 [last_error])
@@ -266,11 +263,11 @@ module Parsanol
 
         # Check consumption
         if consume_all && source.chars_left.positive?
-          context.release_buffer(buffer)
           return context.err(self, source, @extra_error, [last_error])
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        result.pop(result.size - occurrence - 1) if result.size > occurrence + 1
+        ok(result)
       end
     end
   end

data/lib/parsanol/atoms/sequence.rb

@@ -105,7 +105,7 @@ module Parsanol
         ok([:sequence, value])
       end
 
-      # Two-element sequence with buffer pooling
+      # Two-element sequence
       def match_pair(p1, p2, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
         return context.err(self, source, @fail_msg, [v1]) unless success
@@ -113,14 +113,10 @@ module Parsanol
         success, v2 = p2.apply(source, context, consume_all)
         return context.err(self, source, @fail_msg, [v2]) unless success
 
-        buffer = context.acquire_buffer(size: 3)
-        buffer.push(:sequence)
-        buffer.push(v1)
-        buffer.push(v2)
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok([:sequence, v1, v2])
       end
 
-      # Three-element sequence with buffer pooling
+      # Three-element sequence
       def match_triple(p1, p2, p3, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
         return context.err(self, source, @fail_msg, [v1]) unless success
@@ -131,20 +127,16 @@ module Parsanol
         success, v3 = p3.apply(source, context, consume_all)
         return context.err(self, source, @fail_msg, [v3]) unless success
 
-        buffer = context.acquire_buffer(size: 4)
-        buffer.push(:sequence)
-        buffer.push(v1)
-        buffer.push(v2)
-        buffer.push(v3)
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok([:sequence, v1, v2, v3])
       end
 
       # General case for N elements
       def match_general(components, source, context, consume_all)
-        buffer = context.acquire_buffer(size: components.size + 1)
-        buffer.push(:sequence)
+        n = components.size
+        result = Array.new(n + 1)
+        result[0] = :sequence
 
-        last_idx = components.size - 1
+        last_idx = n - 1
         idx = 0
 
         while idx <= last_idx
@@ -152,15 +144,14 @@ module Parsanol
           success, value = components[idx].apply(source, context, must_consume)
 
           unless success
-            context.release_buffer(buffer)
             return context.err(self, source, @fail_msg, [value])
           end
 
-          buffer.push(value)
+          result[idx + 1] = value
           idx += 1
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok(result)
       end
 
       # Merges adjacent string atoms using Rope for efficiency

data/lib/parsanol/native/parser.rb

@@ -8,6 +8,9 @@ module Parsanol
     module Parser
       GRAMMAR_HASH_CACHE = Hash.new
       GRAMMAR_CACHE = Hash.new
+      # structure-hash => Rust-side grammar handle. Keyed by content so a
+      # recycled object_id can never alias a different grammar.
+      HANDLE_CACHE = Hash.new
 
       class << self
         @cached_available = nil
@@ -40,26 +43,51 @@ module Parsanol
 
         # Serialize a Ruby grammar to JSON (cached).
         def serialize_grammar(root_atom)
-          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
-          obj_id = root_atom.object_id
-          cache_key = GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
-          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+          grammar_json(root_atom)
+        end
+
+        # Resolve a Rust-side handle for the grammar, registering it once.
+        # Per-call cost is the memoized structure-hash lookup plus one Hash
+        # access — no JSON marshal, no re-hash inside Rust.
+        def grammar_handle(root_atom)
+          cache_key = grammar_cache_key(root_atom)
+          HANDLE_CACHE[cache_key] ||= Native._register_grammar(grammar_json(root_atom))
+        end
+
+        # Drop a handle whose Rust-side entry no longer exists; the next
+        # grammar_handle call re-registers.
+        def invalidate_handle(handle)
+          HANDLE_CACHE.reject! { |_key, cached| cached == handle }
         end
 
         def clear_cache
           GRAMMAR_HASH_CACHE.clear
           GRAMMAR_CACHE.clear
+          HANDLE_CACHE.clear
         end
 
         def cache_stats
           {
             hash_cache_size: GRAMMAR_HASH_CACHE.size,
             grammar_cache_size: GRAMMAR_CACHE.size,
+            handle_cache_size: HANDLE_CACHE.size,
           }
         end
 
         private
 
+        def grammar_cache_key(root_atom)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          obj_id = root_atom.object_id
+          GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
+        end
+
+        def grammar_json(root_atom)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          cache_key = grammar_cache_key(root_atom)
+          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+        end
+
         def grammar_structure_hash(atom)
           Digest::MD5.hexdigest(atom_structure(atom).to_s)
         end

data/lib/parsanol/native.rb

@@ -38,21 +38,13 @@ module Parsanol
       def parse(grammar, input)
         raise LoadError, "Native parser not available" unless available?
 
-        # Handle both grammar atoms and pre-serialized JSON strings
-        grammar_json = if grammar.is_a?(String)
-                         grammar
-                       else
-                         Parser.serialize_grammar(grammar)
-                       end
+        raw_ast =
+          if grammar.is_a?(String)
+            parse_json_grammar(grammar, input)
+          else
+            parse_atom_grammar(grammar, input)
+          end
 
-        # Use _parse_raw which returns properly tagged Ruby arrays via transform_ast.
-        # The batch format doesn't preserve :repetition/:sequence tags, so we use
-        # the direct FFI path. Apply the Ruby transformer to handle tags correctly.
-        begin
-          raw_ast = _parse_raw(grammar_json, input)
-        rescue RuntimeError => e
-          raise_native_parse_error(e, grammar, input)
-        end
         BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
       end
 
@@ -206,6 +198,33 @@ module Parsanol
         cause = Parsanol::Cause.new(error.message, source, source.bytepos)
         raise Parsanol::ParseFailed.new(cause.to_s, cause)
       end
+
+      # Pre-serialized JSON grammar path (library authors with cached JSON).
+      def parse_json_grammar(grammar_json, input)
+        _parse_raw(grammar_json, input)
+      rescue RuntimeError => e
+        raise_native_parse_error(e, grammar_json, input)
+      end
+
+      # Grammar-atom path: registers once and parses by Rust-side handle,
+      # so steady-state calls marshal no JSON and copy no input string.
+      def parse_atom_grammar(grammar, input)
+        handle = Parser.grammar_handle(grammar)
+
+        begin
+          _parse_handle(handle, input)
+        rescue ArgumentError
+          # Handle dropped Rust-side (e.g. cache cleared): re-register once.
+          Parser.invalidate_handle(handle)
+          begin
+            _parse_handle(Parser.grammar_handle(grammar), input)
+          rescue RuntimeError => e
+            raise_native_parse_error(e, grammar, input)
+          end
+        rescue RuntimeError => e
+          raise_native_parse_error(e, grammar, input)
+        end
+      end
     end
   end
 end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.13"
+  VERSION = "1.3.14"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.13
+  version: 1.3.14
 platform: ruby
 authors:
 - Ribose Inc.