parsanol 1.3.13-arm-linux → 1.3.15-arm-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27b35691df09d839dc38f8cfc72e117b78faedb2e6674171700f89333123e60c
-  data.tar.gz: f76f648e77c782eb676651b1cbdec289333fe56dbac8e26b5ef74572dccaf347
+  metadata.gz: e98e448bc4374451a62e175387759b4db66c90160e5df3204e652c17a751ddd7
+  data.tar.gz: 7f081c2cd9e6fac46cd4e43d15c91a6755af3aad8b01c35b5cb542d190cf2786
 SHA512:
-  metadata.gz: 1e47cf501b7c1a34422aa2a444ab2baf8b32562eae3cc810329547c8b7fea306b4d4dcb3e3615cee78877b9104594541956615995af9aed1f2951bda68272428
-  data.tar.gz: 7d6558cf5ec6a243b06bb1fb0c4b863863d4ebe09890cc475c0d873941426363ba45ed1f3617f5bc75b88e1c96599b04d5e4acd3abb249a8618317726dd57330
+  metadata.gz: 682268466119952e99a678a79270d6fe69a449630665a8e082e7f01f709903f6190552053e6328ec74f47fd452d1b946bedd5b89687a20594debbe33a6e9a2d2
+  data.tar.gz: 386e99e3a11788991e7b1fc5fb0ae55a6c95caa0b60508c62c3f997492c213887135d9d7b4bcb04a72b7d9d8c8bd20a9b20b105b6a2e31cc78b814db32380306

data/lib/parsanol/atoms/base.rb

@@ -30,9 +30,34 @@ module Parsanol
       # @return [Object] the parsed result
       # @raise [Parsanol::ParseFailed] on parse failure
       def parse(source, options = {})
-        input = normalize_input(source)
         must_consume_all = !options[:prefix]
 
+        # Compiled-VM fast path: String input, full-consumption semantics.
+        # On failure (or unsupported grammar / budget exhaustion) fall
+        # through to the interpreter, which also produces the exact
+        # cause-tree diagnostics.
+        if must_consume_all && source.is_a?(String) &&
+            (program = VM.program_for(self))
+          result = VM.run(program, source, true)
+          if result == VM::BAIL
+            # Internal bail: fall back to the interpreter and skip the VM
+            # for this grammar from now on.
+            VM.disable_for!(self)
+          elsif result.first == :heavy
+            # Succeeded but burned >1/8 of the step budget —
+            # heavy-backtracking grammar; the memoizing interpreter is
+            # the better engine from now on.
+            VM.disable_for!(self)
+            return finalize_result(result[1])
+          elsif result.first
+            return finalize_result(result[1])
+          end
+          # Clean [false, nil]: the input does not parse — the interpreter
+          # reparse below produces the detailed cause tree.
+        end
+
+        input = normalize_input(source)
+
         # Initial parse attempt (no error collection)
         success, value = run_with_context(input, nil, must_consume_all)
         return finalize_result(value) if success

data/lib/parsanol/atoms/can_flatten.rb

@@ -65,18 +65,25 @@ module Parsanol
         raise "BUG: Unknown tag #{tag.inspect}."
       end
 
-      # Lisp style fold left where the first element builds the basis for
-      # an inject. Optimized with early return and reduced method calls.
+      # Lisp style fold left where the first non-nil element builds the
+      # basis for an inject. Nil entries are skipped in a single pass so
+      # callers do not need an intermediate #compact.
       #
       def foldl(list)
         len = list.size
-        return "" if len.zero?
-        return list[0] if len == 1 # Fast path for single element
+        i = 0
+        while i < len && list[i].nil?
+          i += 1
+        end
+        return "" if i >= len
+
+        result = list[i]
+        i += 1
+        return result if i >= len # Fast path for single non-nil element
 
-        result = list[0]
-        i = 1
         while i < len
-          result = yield(result, list[i])
+          e = list[i]
+          result = yield(result, e) unless e.nil?
           i += 1
         end
         result
@@ -87,7 +94,7 @@ module Parsanol
       # @api private
       #
       def flatten_sequence(list)
-        foldl(list.compact) do |r, e| # and then merge flat elements
+        foldl(list) do |r, e|
           merge_fold(r, e)
         end
       end
@@ -161,22 +168,65 @@ module Parsanol
         if has_hash
           # If keyed subtrees are in the array, we'll want to discard all
           # strings inbetween. To keep them, name them.
-          return list.select { |e| e.instance_of?(Hash) }
+          hashes = []
+          i = 0
+          while i < len
+            e = list[i]
+            hashes << e if e.instance_of?(Hash)
+            i += 1
+          end
+          return hashes
         end
 
         if has_array
           # If any arrays are nested in this array, flatten all arrays to this
           # level.
-          return list
-              .select { |e| e.instance_of?(Array) }
-              .flatten(1)
+          flat = []
+          i = 0
+          while i < len
+            e = list[i]
+            if e.instance_of?(Array)
+              flat.concat(e)
+            end
+            i += 1
+          end
+          return flat
         end
 
         # Consistent handling of empty lists, when we act on a named result
         return [] if named && list.empty?
 
-        # If there are only strings, concatenate them and return that.
-        foldl(list.compact) { |s, e| s + e }
+        # All-string (or Slice) repetition: join once instead of O(n)
+        # intermediate Slice allocations via repeated #+.
+        join_string_list(list)
+      end
+
+      # Join a list of string-like values into a single Slice (or String),
+      # preserving the first Slice's offset and input. Nils are skipped.
+      def join_string_list(list)
+        first_slice = nil
+        content = nil
+        i = 0
+        len = list.size
+        while i < len
+          e = list[i]
+          unless e.nil?
+            if first_slice.nil? && e.instance_of?(Parsanol::Slice)
+              first_slice = e
+            end
+            piece = e.instance_of?(Parsanol::Slice) ? e.content : e.to_s
+            if content.nil?
+              content = +piece
+            else
+              content << piece
+            end
+          end
+          i += 1
+        end
+        return "" if content.nil?
+        return first_slice.class.new(first_slice.offset, content, first_slice.input) if first_slice
+
+        content
       end
 
       # That annoying warning 'Duplicate subtrees while merging result' comes

data/lib/parsanol/atoms/repetition.rb

@@ -171,16 +171,15 @@ module Parsanol
         context.err_at(self, source, @min_error, source.bytepos, [v3])
       end
 
-      # General repetition with buffer pooling
+      # General repetition: builds the tagged result Array in place so
+      # finalize_result can flatten without an intermediate Buffer/LazyResult
+      # materialization.
       def try_general(source, context, consume_all)
         start_pos = source.bytepos
         occurrence = 0
-
-        # Estimate buffer size
-        estimate = [@max || 10, 10].min
-        buffer = context.acquire_buffer(size: estimate + 1)
-        buffer.push(@result_tag)
-
+        # Pre-size for the tag + a modest run; Array growth is amortized.
+        result = Array.new([@max || 8, 8].min + 1)
+        result[0] = @result_tag
         last_error = nil
 
         loop do
@@ -190,14 +189,13 @@ module Parsanol
           break unless success
 
           occurrence += 1
-          buffer.push(value)
+          result[occurrence] = value
 
           break if @max && occurrence >= @max
         end
 
         # Check minimum bound
         if occurrence < @min
-          context.release_buffer(buffer)
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
                                 [last_error])
@@ -205,11 +203,12 @@ module Parsanol
 
         # Check complete consumption
         if consume_all && source.chars_left.positive?
-          context.release_buffer(buffer)
           return context.err(self, source, @extra_error, [last_error])
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        # Trim to the actual filled length (tag + occurrence matches).
+        result.pop(result.size - occurrence - 1) if result.size > occurrence + 1
+        ok(result)
       end
 
       # Tree memoization for GPEG-style caching
@@ -227,23 +226,21 @@ module Parsanol
 
         # Parse and cache
         occurrence = 0
-        estimate = [@max || 10, 10].min
-        buffer = context.acquire_buffer(size: estimate + 1)
-        buffer.push(@result_tag)
+        result = Array.new([@max || 8, 8].min + 1)
+        result[0] = @result_tag
 
         positions = context.acquire_array
         positions << start_pos
         last_error = nil
 
         loop do
-          source.bytepos
           success, value = @parslet.apply(source, context, false)
           last_error = value
 
           break unless success
 
           occurrence += 1
-          buffer.push(value)
+          result[occurrence] = value
           positions << source.bytepos
 
           break if @max && occurrence >= @max
@@ -252,13 +249,13 @@ module Parsanol
         # Cache successful prefix
         if occurrence.positive?
           end_pos = positions[occurrence]
-          context.store_tree_memo(cache_key, start_pos, buffer.to_a[1..],
-                                  end_pos)
+          context.store_tree_memo(cache_key, start_pos,
+                                  result[1, occurrence], end_pos)
         end
+        context.release_array(positions) if context.respond_to?(:release_array)
 
         # Check minimum
         if occurrence < @min
-          context.release_buffer(buffer)
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
                                 [last_error])
@@ -266,11 +263,11 @@ module Parsanol
 
         # Check consumption
         if consume_all && source.chars_left.positive?
-          context.release_buffer(buffer)
           return context.err(self, source, @extra_error, [last_error])
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        result.pop(result.size - occurrence - 1) if result.size > occurrence + 1
+        ok(result)
       end
     end
   end

data/lib/parsanol/atoms/sequence.rb

@@ -105,7 +105,7 @@ module Parsanol
         ok([:sequence, value])
       end
 
-      # Two-element sequence with buffer pooling
+      # Two-element sequence
       def match_pair(p1, p2, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
         return context.err(self, source, @fail_msg, [v1]) unless success
@@ -113,14 +113,10 @@ module Parsanol
         success, v2 = p2.apply(source, context, consume_all)
         return context.err(self, source, @fail_msg, [v2]) unless success
 
-        buffer = context.acquire_buffer(size: 3)
-        buffer.push(:sequence)
-        buffer.push(v1)
-        buffer.push(v2)
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok([:sequence, v1, v2])
       end
 
-      # Three-element sequence with buffer pooling
+      # Three-element sequence
       def match_triple(p1, p2, p3, source, context, consume_all)
         success, v1 = p1.apply(source, context, false)
         return context.err(self, source, @fail_msg, [v1]) unless success
@@ -131,20 +127,16 @@ module Parsanol
         success, v3 = p3.apply(source, context, consume_all)
         return context.err(self, source, @fail_msg, [v3]) unless success
 
-        buffer = context.acquire_buffer(size: 4)
-        buffer.push(:sequence)
-        buffer.push(v1)
-        buffer.push(v2)
-        buffer.push(v3)
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok([:sequence, v1, v2, v3])
       end
 
       # General case for N elements
       def match_general(components, source, context, consume_all)
-        buffer = context.acquire_buffer(size: components.size + 1)
-        buffer.push(:sequence)
+        n = components.size
+        result = Array.new(n + 1)
+        result[0] = :sequence
 
-        last_idx = components.size - 1
+        last_idx = n - 1
         idx = 0
 
         while idx <= last_idx
@@ -152,15 +144,14 @@ module Parsanol
           success, value = components[idx].apply(source, context, must_consume)
 
           unless success
-            context.release_buffer(buffer)
             return context.err(self, source, @fail_msg, [value])
           end
 
-          buffer.push(value)
+          result[idx + 1] = value
           idx += 1
         end
 
-        ok(Parsanol::LazyResult.new(buffer, context))
+        ok(result)
       end
 
       # Merges adjacent string atoms using Rope for efficiency

data/lib/parsanol/native/parser.rb

@@ -8,6 +8,9 @@ module Parsanol
     module Parser
       GRAMMAR_HASH_CACHE = Hash.new
       GRAMMAR_CACHE = Hash.new
+      # structure-hash => Rust-side grammar handle. Keyed by content so a
+      # recycled object_id can never alias a different grammar.
+      HANDLE_CACHE = Hash.new
 
       class << self
         @cached_available = nil
@@ -40,26 +43,51 @@ module Parsanol
 
         # Serialize a Ruby grammar to JSON (cached).
         def serialize_grammar(root_atom)
-          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
-          obj_id = root_atom.object_id
-          cache_key = GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
-          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+          grammar_json(root_atom)
+        end
+
+        # Resolve a Rust-side handle for the grammar, registering it once.
+        # Per-call cost is the memoized structure-hash lookup plus one Hash
+        # access — no JSON marshal, no re-hash inside Rust.
+        def grammar_handle(root_atom)
+          cache_key = grammar_cache_key(root_atom)
+          HANDLE_CACHE[cache_key] ||= Native._register_grammar(grammar_json(root_atom))
+        end
+
+        # Drop a handle whose Rust-side entry no longer exists; the next
+        # grammar_handle call re-registers.
+        def invalidate_handle(handle)
+          HANDLE_CACHE.reject! { |_key, cached| cached == handle }
         end
 
         def clear_cache
           GRAMMAR_HASH_CACHE.clear
           GRAMMAR_CACHE.clear
+          HANDLE_CACHE.clear
         end
 
         def cache_stats
           {
             hash_cache_size: GRAMMAR_HASH_CACHE.size,
             grammar_cache_size: GRAMMAR_CACHE.size,
+            handle_cache_size: HANDLE_CACHE.size,
           }
         end
 
         private
 
+        def grammar_cache_key(root_atom)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          obj_id = root_atom.object_id
+          GRAMMAR_HASH_CACHE[obj_id] ||= grammar_structure_hash(root_atom)
+        end
+
+        def grammar_json(root_atom)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          cache_key = grammar_cache_key(root_atom)
+          GRAMMAR_CACHE[cache_key] ||= GrammarSerializer.serialize(root_atom)
+        end
+
         def grammar_structure_hash(atom)
           Digest::MD5.hexdigest(atom_structure(atom).to_s)
         end

data/lib/parsanol/native/transformer.rb

@@ -132,9 +132,11 @@ module Parsanol
         # Transform the value
         transformed = transform(value, named: true)
 
-        # Check if value is a tagged repetition from native parser
+        # Check if value is a tagged repetition from native parser.
+        # The Rust handle path tags with Symbols, the batch decoder with
+        # ":repetition" Strings — accept both.
         is_tagged_repetition = value.is_a?(Array) && !value.empty? &&
-          value.first.is_a?(String) && value.first == REPETITION_TAG
+          [REPETITION_SYM, REPETITION_TAG].include?(value.first)
 
         # Check RAW value for repetition pattern BEFORE transformation
         # Array with items that all have the parent key
@@ -203,13 +205,12 @@ module Parsanol
           # Empty array from repetition stays as empty array
           if transformed.empty?
             { sym_key => EMPTY_ARRAY }
-          # Check if items already have the same key (avoid double-wrapping)
-          elsif transformed.all? do |item|
-            item.is_a?(Hash) && item.key?(sym_key)
-          end
+          # Hash items already carry their own capture names — a
+          # repetition of named captures keeps them as-is (parslet
+          # semantics); only unnamed items get the parent name per item.
+          elsif transformed.all?(Hash)
             { sym_key => transformed }
           else
-            # Wrap each item with the name
             { sym_key => transformed.map { |item| { sym_key => item } } }
           end
         elsif transformed.is_a?(::Parsanol::Slice) && transformed.empty?

data/lib/parsanol/native.rb

@@ -38,22 +38,14 @@ module Parsanol
       def parse(grammar, input)
         raise LoadError, "Native parser not available" unless available?
 
-        # Handle both grammar atoms and pre-serialized JSON strings
-        grammar_json = if grammar.is_a?(String)
-                         grammar
-                       else
-                         Parser.serialize_grammar(grammar)
-                       end
-
-        # Use _parse_raw which returns properly tagged Ruby arrays via transform_ast.
-        # The batch format doesn't preserve :repetition/:sequence tags, so we use
-        # the direct FFI path. Apply the Ruby transformer to handle tags correctly.
-        begin
-          raw_ast = _parse_raw(grammar_json, input)
-        rescue RuntimeError => e
-          raise_native_parse_error(e, grammar, input)
+        # Both sub-methods return the final decoded tree; on native failure
+        # they fall back to the pure-Ruby parser, whose result is already
+        # final and must not be transformed again.
+        if grammar.is_a?(String)
+          parse_json_grammar(grammar, input)
+        else
+          parse_atom_grammar(grammar, input)
         end
-        BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
       end
 
       # Memory-bounded parsing without packrat cache.
@@ -73,12 +65,14 @@ module Parsanol
                          Parser.serialize_grammar(grammar)
                        end
 
+        # Decode here, not around the fallback: raise_native_parse_error
+        # returns the Ruby parser's already-final tree on success.
         begin
-          raw_ast = _parse_fresh_raw(grammar_json, input)
+          BatchDecoder.decode_and_flatten(_parse_fresh_raw(grammar_json, input),
+                                          input, Parsanol::Slice)
         rescue RuntimeError => e
           raise_native_parse_error(e, grammar, input)
         end
-        BatchDecoder.decode_and_flatten(raw_ast, input, Parsanol::Slice)
       end
 
       # Parse and return RAW AST without transformation.
@@ -206,6 +200,38 @@ module Parsanol
         cause = Parsanol::Cause.new(error.message, source, source.bytepos)
         raise Parsanol::ParseFailed.new(cause.to_s, cause)
       end
+
+      # Pre-serialized JSON grammar path (library authors with cached JSON).
+      def parse_json_grammar(grammar_json, input)
+        BatchDecoder.decode_and_flatten(_parse_raw(grammar_json, input),
+                                        input, Parsanol::Slice)
+      rescue RuntimeError => e
+        raise_native_parse_error(e, grammar_json, input)
+      end
+
+      # Grammar-atom path: registers once and parses by Rust-side handle,
+      # so steady-state calls marshal no JSON and copy no input string.
+      def parse_atom_grammar(grammar, input)
+        handle = Parser.grammar_handle(grammar)
+
+        begin
+          BatchDecoder.decode_and_flatten(_parse_handle(handle, input),
+                                          input, Parsanol::Slice)
+        rescue ArgumentError
+          # Handle dropped Rust-side (e.g. cache cleared): re-register once.
+          Parser.invalidate_handle(handle)
+          begin
+            BatchDecoder.decode_and_flatten(
+              _parse_handle(Parser.grammar_handle(grammar), input),
+              input, Parsanol::Slice
+            )
+          rescue RuntimeError => e
+            raise_native_parse_error(e, grammar, input)
+          end
+        rescue RuntimeError => e
+          raise_native_parse_error(e, grammar, input)
+        end
+      end
     end
   end
 end

data/lib/parsanol/parser.rb

@@ -109,32 +109,36 @@ module Parsanol
     #   result[:name].to_s           # => "hello"
     #
     def parse(input, mode_or_opts = {}, **kwargs)
-      if mode_or_opts.is_a?(Hash) && !kwargs.key?(:mode)
-        # Legacy API: parse(input, options={})
-        merged = mode_or_opts.merge(kwargs)
-        if Parsanol::Native.available? && !merged.key?(:prefix) &&
-            !merged.key?(:reporter)
-          # Native backend by default; falls back to pure Ruby inside
-          # parse_native when the extension is missing.
-          parse_native(input, merged)
+      opts =
+        if mode_or_opts.is_a?(Hash)
+          mode_or_opts.merge(kwargs)
+        elsif mode_or_opts.nil?
+          kwargs
         else
-          super(input, merged)
+          # New API: parse(input, :mode, **options)
+          kwargs[:mode] = mode_or_opts
+          kwargs
         end
+      # parse(input, {mode: :ruby, ...}) is the same call shape a subclass
+      # forwarding options via super produces; :mode must be honored
+      # wherever it appears, or those callers silently get the native path.
+      mode = opts.delete(:mode) ||
+        (if Parsanol::Native.available? && !opts.key?(:prefix) &&
+              !opts.key?(:reporter)
+           :native
+         else
+           :ruby
+         end)
+      case mode
+      when :ruby
+        super(input, opts)
+      when :native
+        parse_native(input, opts)
+      when :json
+        parse_json(input, opts)
       else
-        # New API: parse(input, mode:, **options)
-        mode = kwargs.delete(:mode) ||
-          (Parsanol::Native.available? ? :native : :ruby)
-        case mode
-        when :ruby
-          super(input, kwargs)
-        when :native
-          parse_native(input, kwargs)
-        when :json
-          parse_json(input, kwargs)
-        else
-          raise ArgumentError,
-                "Unknown mode: #{mode}. Valid modes: :ruby, :native, :json"
-        end
+        raise ArgumentError,
+              "Unknown mode: #{mode}. Valid modes: :ruby, :native, :json"
       end
     end
 

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.13"
+  VERSION = "1.3.15"
 end