parsanol 1.3.15 → 1.3.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 222de318e9cac8557e165fef1471947f25932448540a63756735581fa79f455c
-  data.tar.gz: e6659f5259e0e76cf701a0630ed50a48cb4a217d3d6a6ebc5ab703ab2088f062
+  metadata.gz: 99c30bd8fcd284b2fe099ffe93f0e462d182906788f87b4bb91b6828111e0b66
+  data.tar.gz: e83f0633dd370b5b18c0b8a18c37dc22ce6c4a7ce5d12aa482a8656f545dc1c9
 SHA512:
-  metadata.gz: a20812f786fde6730b244e64f322ebd8d5a2a3d59ea9a03743cf8a0f12142f9d4efe2b01c3f3e29a19329871d1290364eac1983048869c14784576f1383b983e
-  data.tar.gz: 0bd90a5bab7e78b97c5e63abdb3df077cc8372f462a296d798abd28f6718857b3b96403e568267460a07c873754b00b372a14dc802dd415a2abdaae41d34f1e6
+  metadata.gz: 7b6aa212fcf424baf769942c3626f71fd790ffdffe44df79fe79e33aa54bdc8e64c0d8a3abf989e9b207e3a0741d7061afae3dd3600750a0422b34da03d44e80
+  data.tar.gz: b3386af97c1247146ed6d4f1218466fc02834c0e69bd538edde652fa2e3dcb22226f17760655439394f2482c400478b14b424101eb7927ae3b85e3a829de9f57

data/lib/parsanol/atoms/base.rb

@@ -35,20 +35,15 @@ module Parsanol
         # Compiled-VM fast path: String input, full-consumption semantics.
         # On failure (or unsupported grammar / budget exhaustion) fall
         # through to the interpreter, which also produces the exact
-        # cause-tree diagnostics.
+        # cause-tree diagnostics. run_for memoizes heavy-backtracking
+        # grammars transparently.
         if must_consume_all && source.is_a?(String) &&
             (program = VM.program_for(self))
-          result = VM.run(program, source, true)
+          result = VM.run_for(self, program, source, true)
           if result == VM::BAIL
             # Internal bail: fall back to the interpreter and skip the VM
             # for this grammar from now on.
             VM.disable_for!(self)
-          elsif result.first == :heavy
-            # Succeeded but burned >1/8 of the step budget —
-            # heavy-backtracking grammar; the memoizing interpreter is
-            # the better engine from now on.
-            VM.disable_for!(self)
-            return finalize_result(result[1])
           elsif result.first
             return finalize_result(result[1])
           end
@@ -153,6 +148,13 @@ module Parsanol
         to_s(TOP)
       end
 
+      # Finalizes result by flattening. Public: the native error
+      # fallback (both extension and ffi tiers) calls it cross-object to
+      # finalize a recovered tree.
+      def finalize_result(value)
+        flatten(value)
+      end
+
       protected
 
       # Pre-allocated constant result tuples
@@ -223,11 +225,6 @@ module Parsanol
 
         cause.raise
       end
-
-      # Finalizes result by flattening.
-      def finalize_result(value)
-        flatten(value)
-      end
     end
   end
 end

data/lib/parsanol/native/ffi.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Native
+    # Rust engine for runtimes that cannot load C-API extensions
+    # (JRuby, TruffleRuby, or an MRI whose platform gem is absent and
+    # cannot build one). Binds the parsanol cdylib's C ABI through the
+    # `ffi` gem: grammars register once into Rust-side handles keyed by
+    # serialized structure; results cross the boundary as flat-u64 batch
+    # arrays decoded by the same BatchDecoder used elsewhere.
+    module Ffi
+      class Error < StandardError
+      end
+
+      class << self
+        @available = nil
+
+        def available?
+          return @available unless @available.nil?
+
+          @available = begin
+            require "ffi"
+            lib = locate_library
+            lib && bind_library(lib)
+          rescue LoadError, StandardError
+            false
+          end
+        end
+
+        # Registers (once, cached) and parses. +grammar+ is a grammar
+        # atom or a pre-serialized JSON string. Mirrors Native.parse's
+        # contract, including the single reporter-pass fallback to the
+        # pure-Ruby engine for parslet-compatible failure errors.
+        def parse(grammar, input)
+          unless available? && @binding
+            raise Error, "parsanol FFI library not available"
+          end
+
+          if grammar.is_a?(String)
+            json = grammar
+            atom = nil
+          else
+            json = ::Parsanol::Native::Parser.serialize_grammar(grammar)
+            atom = grammar
+          end
+          handle = ((@handles ||= {})[json] ||= @binding.c_register(json))
+          raise Error, "grammar registration failed" if handle.zero?
+
+          written = parse_into(handle, input)
+          if written.positive?
+            return ::Parsanol::Native::BatchDecoder.decode_and_flatten(
+              @buffer.read_array_of_uint64(written), input, ::Parsanol::Slice
+            )
+          end
+
+          message = @binding.c_last_error.to_s
+          if atom
+            return ::Parsanol::Native.raise_native_parse_error(
+              RuntimeError.new(message), atom, input
+            )
+          end
+
+          raise Error, "parse failed: #{message}"
+        end
+
+        private
+
+        # Search order: explicit override, gem-vendored cdylib next to
+        # this file, then the system linker path.
+        def locate_library
+          candidates = []
+          env = ENV.fetch("PARSANOL_FFI_LIB", nil)
+          candidates << env if env && !env.empty?
+          here = File.dirname(__FILE__)
+          %w[libparsanol.dylib libparsanol.so parsanol.dll].each do |name|
+            path = File.expand_path(name, here)
+            candidates << path if File.file?(path)
+          end
+          candidates << "parsanol"
+          candidates.each do |cand|
+            return cand if File.file?(cand)
+
+            begin
+              FFI::DynamicLibrary.open(cand, FFI::DynamicLibrary::RTLD_NOW)
+              return cand
+            rescue StandardError
+              next
+            end
+          end
+          nil
+        end
+
+        def bind_library(path) # rubocop:disable Naming/PredicateMethod -- binds and reports success
+          binding_mod = Module.new do
+            extend FFI::Library
+
+            ffi_lib path
+            attach_function :c_register, :parsanol_c_register,
+                            %i[string], :uint64
+            attach_function :c_parse, :parsanol_c_parse,
+                            %i[uint64 string pointer size_t], :long_long
+            attach_function :c_last_error, :parsanol_c_last_error,
+                            [], :string
+            attach_function :c_release, :parsanol_c_release,
+                            %i[uint64], :void
+          end
+          # Probe the symbols now so a mismatched library fails loudly
+          # at availability time, not at first parse.
+          probe = binding_mod.c_last_error
+          @binding = binding_mod
+          @buffer = nil
+          !probe.nil? || true
+        end
+
+        # Two-call buffer protocol: cap=0 asks for the needed size, then
+        # one allocation carries the whole batch.
+        def parse_into(handle, input)
+          needed = @binding.c_parse(handle, input, nil, 0)
+          return needed unless needed.negative?
+
+          @buffer = FFI::MemoryPointer.new(:uint64, -needed)
+          @binding.c_parse(handle, input, @buffer, -needed)
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/native/parser.rb

@@ -18,7 +18,7 @@ module Parsanol
         def available?
           return @cached_available unless @cached_available.nil?
 
-          @cached_available = begin
+          @ext_loaded = begin
             # Try versioned path first (released gem), then non-versioned (local dev)
             ruby_version = RUBY_VERSION.split(".").take(2).join(".")
             begin
@@ -30,6 +30,13 @@ module Parsanol
           rescue LoadError
             false
           end
+          @cached_available = @ext_loaded || Ffi.available?
+        end
+
+        # True when the MRI C-API extension is loaded; false when the
+        # engine runs through the ffi-gem cdylib tier instead.
+        def extension_loaded?
+          @ext_loaded ? true : false
         end
 
         # Parse input with a Ruby grammar, returning clean AST.

data/lib/parsanol/native.rb

@@ -10,6 +10,8 @@ require "parsanol/native/batch_decoder"
 
 module Parsanol
   module Native
+    # ffi-gem cdylib tier: lazy — only loaded when the MRI extension is absent.
+    autoload :Ffi, "parsanol/native/ffi"
     class << self
       # Check if native extension is available
       def available?
@@ -38,6 +40,10 @@ module Parsanol
       def parse(grammar, input)
         raise LoadError, "Native parser not available" unless available?
 
+        # ffi-gem tier (JRuby/TruffleRuby/no-binary MRI): same contract,
+        # batch-decoded results, single reporter-pass error fallback.
+        return Ffi.parse(grammar, input) unless Parser.extension_loaded?
+
         # Both sub-methods return the final decoded tree; on native failure
         # they fall back to the pure-Ruby parser, whose result is already
         # final and must not be transformed again.
@@ -194,7 +200,20 @@ module Parsanol
       # pre-serialized JSON grammars the native message is wrapped in a
       # Parsanol::ParseFailed directly.
       def raise_native_parse_error(error, grammar, input)
-        return grammar.parse(input) if grammar.respond_to?(:parse)
+        if grammar.respond_to?(:parse)
+          # One interpreter pass with the error reporter attached covers
+          # both jobs: a success means the native backend could not
+          # express the grammar (recover the tree), a failure raises the
+          # parslet-compatible cause tree. The native backend has
+          # already failed, so a separate plain attempt first would be a
+          # wasted parse.
+          reporter = Parsanol::ErrorReporter::Tree.new
+          source = Parsanol::Source.new(input)
+          success, value = grammar.run_with_context(source, reporter, true)
+          return grammar.finalize_result(value) if success
+
+          value.raise
+        end
 
         source = Parsanol::Source.new(input)
         cause = Parsanol::Cause.new(error.message, source, source.bytepos)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.15"
+  VERSION = "1.3.17"
 end

data/lib/parsanol/vm.rb

@@ -71,9 +71,18 @@ module Parsanol
     # and may skip the VM for this grammar afterwards. A clean
     # [false, nil] means the input genuinely does not parse.
     BAIL = Object.new.freeze
+    # Step budget exhausted on the naive pass — internal to #run, which
+    # retries once with packrat memoization before giving up.
+    BUDGET = Object.new.freeze
+    # Memo marker for a subroutine currently being evaluated at a given
+    # position. Re-entry (left recursion) fails that path instead of
+    # looping forever.
+    PENDING = Object.new.freeze
 
     STEP_BUDGET_FACTOR = 200
     STEP_BUDGET_FIXED = 10_000
+    MEMO_BUDGET_FACTOR = 4_096
+    MEMO_BUDGET_CELLS_FACTOR = 16
 
     class << self
       @programs = {}
@@ -108,13 +117,27 @@ module Parsanol
 
       def clear_program_cache
         @programs&.clear
+        @heavy&.clear
       end
 
       # Compiles the grammar rooted at +atom+. Returns the flat program
       # Array, or nil when the grammar uses unsupported atoms.
       def compile(atom)
+        program = compile_with_inline(atom, true)
+        # Inlining duplicates every non-recursive rule body at each
+        # reference site; grammars with many cross-referencing rules
+        # explode past the program cap even though the atom tree is
+        # modest. Recompiling with inlining off (each rule a CALL/RET
+        # subroutine) keeps the program O(atoms) at a small per-rule
+        # dispatch cost — far better than refusing the grammar outright.
+        return program unless program == :oversize
+
+        compile_with_inline(atom, false)
+      end
+
+      def compile_with_inline(atom, inline)
         root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
-        compiler = Compiler.new
+        compiler = Compiler.new(inline: inline)
         return nil unless compiler.compile_atom(root, true)
 
         # HALT terminates the MAIN program; subroutines follow it so the
@@ -128,8 +151,58 @@ module Parsanol
       # Executes a compiled program. Returns [true, value] on success;
       # [false, nil] on failure or budget exhaustion — callers fall back
       # to the interpreter, which reproduces exact diagnostics.
+      #
+      # Heavy-backtracking grammars get a second chance: the naive pass
+      # runs without memoization (zero overhead on the happy path), and
+      # only if it blows the step budget does a memoized pass run. The
+      # memo converts repeated rule+position work into single entries, so
+      # grammars that the interpreter needed (slow) memoization for now
+      # run at VM speed.
       def run(program, input, consume_all)
-        Executor.new(program, input, consume_all).execute
+        executor = Executor.new(program, input, consume_all)
+        result = executor.execute
+        if result.equal?(BUDGET)
+          executor = Executor.new(program, input, consume_all, memoize: true)
+          result = executor.execute
+          result = BAIL if result.equal?(BUDGET)
+        end
+        result
+      end
+
+      # Grammar-aware entry used by Base#parse. Grammars that have shown
+      # heavy-backtracking behavior (budget bust, or a success that
+      # burned the step-density threshold) memoize from the first
+      # instruction on subsequent parses, skipping the doomed naive
+      # pass; a heavy success is retried memoized in the same call so
+      # the VM keeps the grammar instead of surrendering it to the
+      # interpreter.
+      def run_for(atom, program, input, consume_all)
+        root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
+        id = root.object_id
+        heavy = (@heavy ||= {})
+        if heavy[id]
+          result = Executor.new(program, input, consume_all,
+                                memoize: true).execute
+          return BAIL if result.equal?(BUDGET)
+
+          return result
+        end
+
+        result = Executor.new(program, input, consume_all).execute
+        if result.equal?(BUDGET) ||
+            (result.is_a?(Array) && result.first == :heavy)
+          heavy[id] = true
+          memo_result = Executor.new(program, input, consume_all,
+                                     memoize: true).execute
+          # A heavy naive success already answered; the memo pass only
+          # proves the grammar stays VM-viable (BUDGET would bail to the
+          # interpreter). When the naive pass never answered, the memo
+          # pass's outcome is the result.
+          return BAIL if memo_result.equal?(BUDGET)
+
+          result = memo_result if result.equal?(BUDGET)
+        end
+        result
       end
 
       # Converts the VM value tree into the interpreter's value tree.
@@ -159,8 +232,9 @@ module Parsanol
     # as subroutines, keyed by [body object_id, consume_all] so spine
     # sites (consume_all=true) and inner sites stay semantically distinct.
     class Compiler
-      def initialize
+      def initialize(inline: true)
         @ops = []
+        @inline = inline
         @subs = {}      # [obj_id, flag] => pc or :pending
         @pending = {}   # [obj_id, flag] => [[call_idx, body], ...]
       end
@@ -168,7 +242,7 @@ module Parsanol
       attr_reader :ops
 
       def to_program
-        return nil if @ops.size > MAX_PROGRAM
+        return :oversize if @ops.size > MAX_PROGRAM
 
         @ops.flatten!(1)
       end
@@ -244,7 +318,21 @@ module Parsanol
 
           emit(DROP)
           self
-          # Dynamic, Capture, Scope, Cut, Custom, Infix, unknown
+        when Parsanol::Atoms::Scope
+          # Scope only affects capture state; the result tree is the
+          # inner atom's tree unchanged. The block is pure DSL evaluated
+          # once at compile time. Any Capture/Dynamic inside still fails
+          # compilation on its own, so passthrough cannot diverge.
+          inner = begin
+            atom.block.call
+          rescue StandardError
+            nil
+          end
+          return nil if inner.nil?
+          return nil unless compile_atom(inner, consume_all)
+
+          self
+          # Dynamic, Capture, Cut, Custom, Infix, unknown
         end
       end
 
@@ -401,7 +489,7 @@ module Parsanol
         # round trip entirely. A body currently being compiled (directly
         # or indirectly) is recursive and must stay a subroutine.
         # rubocop:disable Lint/HashCompareByIdentity -- object_id keys; would otherwise pin every atom alive
-        unless (@compiling ||= {})[body.object_id]
+        if @inline && !(@compiling ||= {})[body.object_id]
           @compiling[body.object_id] = true
           # rubocop:enable Lint/HashCompareByIdentity
           result = compile_atom(body, consume_all)
@@ -456,10 +544,11 @@ module Parsanol
     # (with its adaptive memoization) is the better engine; #run reports
     # this via :heavy so Base#parse can skip the VM for the grammar.
     class Executor
-      def initialize(program, input, consume_all)
+      def initialize(program, input, consume_all, memoize: false)
         @ops = program
         @input = input
         @consume_all = consume_all
+        @memoize = memoize
       end
 
       def execute # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/BlockNesting -- single dispatch loop; hot path
@@ -471,7 +560,15 @@ module Parsanol
         # interpreter's Source#matches? semantics exactly.
         scanner = StringScanner.new(input)
         n = bytes.size
-        budget = (STEP_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+        # The memoized pass is polynomial (bounded by distinct rule ×
+        # position pairs); give it room proportional to grammar + input so
+        # packrat coverage of failing inputs isn't cut short.
+        budget = if @memoize
+                   (MEMO_BUDGET_CELLS_FACTOR * ops.size) +
+                     (MEMO_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+                 else
+                   (STEP_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+                 end
         steps = 0
 
         pc = 0
@@ -481,13 +578,24 @@ module Parsanol
         frames = []
         calls = []
 
+        memo = @memoize ? {} : nil
+        memo_stack = []
+        trace = ENV["VM_TRACE"] ? [] : nil
+
         # rubocop:disable-next Metrics/BlockLength -- the dispatch loop IS execute
         loop do
           steps += 1
-          return BAIL if steps > budget
+          if steps > budget
+            if trace
+              warn "BUDGET at steps=#{steps} pc=#{pc} pos=#{pos} rstack=#{rstack.size} bt=#{bt.size} calls=#{calls.size}"
+              warn "trace tail: #{trace.last(30).inspect}"
+            end
+            return BUDGET
+          end
+          trace << [pc, pos] if trace
 
           if pc == FAIL
-            pc, pos = unwind(bt, rstack, frames, calls)
+            pc, pos = unwind(bt, rstack, frames, calls, memo, memo_stack)
             return [false, nil] if pc == :fail
             return BAIL if pc == :bail
 
@@ -579,11 +687,45 @@ module Parsanol
             rstack << VM::NamedValue.new(ops[pc + 1], rstack.pop)
             pc += 4
           when CALL
+            sub = ops[pc + 1]
+            if memo
+              table = (memo[sub] ||= {})
+              hit = table[pos]
+              if hit
+                if hit.equal?(PENDING) || hit.equal?(:fail)
+                  # :fail — the body already failed at this position;
+                  # replay the failure without re-running it.
+                  # PENDING — re-entry at the same rule+position (left
+                  # recursion); the interpreter loops forever here, so
+                  # failing this path keeps the VM bounded.
+                  pc = FAIL
+                  next
+                end
+                # Memo hit: replay the subroutine's stack effect without
+                # executing it. Values are immutable (packed spans,
+                # NamedValue, frozen-shape Arrays), so sharing is safe.
+                rstack.concat(hit[1])
+                pos = hit[0]
+                pc += 4
+                next
+              end
+
+              table[pos] = PENDING
+              # Lockstep with calls: unwind rolls both back by clen.
+              memo_stack << sub << pos << rstack.size
+            end
             calls << (pc + 4)
-            pc = ops[pc + 1]
+            pc = sub
           when RET
             pc = calls.pop
             return BAIL if pc.nil?
+
+            if memo
+              mlen = memo_stack.pop
+              mpos = memo_stack.pop
+              msub = memo_stack.pop
+              memo[msub][mpos] = [pos, rstack[mlen..]]
+            end
           when LOOK_POS
             bt << ops[pc + 1] << pos << rstack.size << frames.size << calls.size << K_LOOK_FAIL
             pc += 4
@@ -735,6 +877,8 @@ module Parsanol
                   values << [:maybe]
                 end
               when RUN_RE
+                elem_start = pos
+                count = 0
                 loop do
                   b = bytes[pos]
                   m = if kb && b && b < 128
@@ -746,12 +890,19 @@ module Parsanol
                       end
                   break unless m
 
+                  count += 1
                   w = CHAR_WIDTH[b]
                   w = char_width_slow(input, pos) if w.zero?
                   values << ((pos << 20) | w)
                   pos += w
                 end
+                if count < kc
+                  pos = elem_start
+                  failed = true
+                end
               when RUN_STR
+                elem_start = pos
+                count = 0
                 loop do
                   i = 0
                   ln = kb
@@ -765,9 +916,14 @@ module Parsanol
                   end
                   break unless m
 
+                  count += 1
                   values << ((pos << 20) | ln)
                   pos += ln
                 end
+                if count < kc
+                  pos = elem_start
+                  failed = true
+                end
               end
               break if failed
 
@@ -878,6 +1034,10 @@ module Parsanol
             value = VM.materialize(rstack[0], input)
             # More than ~100 steps per input byte means heavy
             # backtracking; the memoizing interpreter wins there.
+            if @memoize
+              return [true, value]
+            end
+
             return steps > (n << 9) + 1000 ? [:heavy, value] : [true, value]
           else
             return BAIL
@@ -890,7 +1050,7 @@ module Parsanol
       # Unwind one backtrack entry. Returns [pc, pos] or :fail. Repetition
       # entries with count >= min complete their tagged array and jump to
       # the exit continuation with the last iteration end position.
-      def unwind(bt, rstack, frames, calls) # rubocop:disable Naming/MethodParameterName -- stack register names
+      def unwind(bt, rstack, frames, calls, memo, memo_stack) # rubocop:disable Naming/MethodParameterName -- stack register names
         return :fail if bt.empty?
 
         # Everything above this entry belongs to constructs being
@@ -910,6 +1070,24 @@ module Parsanol
 
         rstack.slice!(rlen..) if rstack.size > rlen
         calls.slice!(clen..) if calls.size > clen
+        if memo
+          # memo_stack carries three slots per live call, so the frames of
+          # every call abandoned by this unwind are the tail above 3*clen.
+          # Every abandoned frame is a call whose body failed — cache that
+          # failure too (full packrat memoization), so later explorations
+          # that reach the same rule+position fail immediately instead of
+          # re-running the body. Grammar inputs that don't parse explore
+          # every alternative; without failure entries those explorations
+          # repeat exponentially and blow the step budget.
+          floor = clen * 3
+          while memo_stack.size > floor
+            memo_stack.pop
+            mpos = memo_stack.pop
+            msub = memo_stack.pop
+            table = memo[msub]
+            table[mpos] = :fail if table
+          end
+        end
 
         case kind
         when K_ALT
@@ -930,7 +1108,7 @@ module Parsanol
             check_full = @ops[epc + 2]
             if check_full == 1 && end_pos != @input.bytesize
               frames.slice!(cbase..)
-              unwind(bt, rstack, frames, calls)
+              unwind(bt, rstack, frames, calls, memo, memo_stack)
             else
               values = rstack.pop(rstack.size - rbase)
               values.unshift(tag)
@@ -940,11 +1118,11 @@ module Parsanol
             end
           else
             frames.slice!(cbase..)
-            unwind(bt, rstack, frames, calls)
+            unwind(bt, rstack, frames, calls, memo, memo_stack)
           end
         when K_LOOK_FAIL
           frames.slice!(cbase..) if frames.size > cbase
-          unwind(bt, rstack, frames, calls)
+          unwind(bt, rstack, frames, calls, memo, memo_stack)
         when K_NEG
           # Negative lookahead body failed -> lookahead succeeds.
           frames.slice!(cbase..) if frames.size > cbase

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.15
+  version: 1.3.17
 platform: ruby
 authors:
 - Ribose Inc.
@@ -215,6 +215,7 @@ files:
 - lib/parsanol/native.rb
 - lib/parsanol/native/batch_decoder.rb
 - lib/parsanol/native/dynamic.rb
+- lib/parsanol/native/ffi.rb
 - lib/parsanol/native/parser.rb
 - lib/parsanol/native/serializer.rb
 - lib/parsanol/native/transformer.rb