parsanol 1.3.15-aarch64-linux → 1.3.16-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: acc35902954056dfb6f92c006858f8323797b76dfdfcfd217d86ed2feb6f4341
-  data.tar.gz: e11548e00a5386bf22cbfd5ce6cd61ac0d943edd76bc08c7bfce08994eb0e103
+  metadata.gz: 47f7be12e8a16ae18e07d621fc699bd49d55367c4e17a6f43c3ba1a00f12bfc7
+  data.tar.gz: 3877f2d6c8d48495efd798abf72bd96bbc7e6fc643cf6de78ca0a6b589445b37
 SHA512:
-  metadata.gz: 6b27c78d0c3417805dba427584c3247b870d31079a1b901930cd8a6343c803cabd0f071e70de382bc3587a191f5f92333a7e6b5c65693ab4419117ae4da4fac5
-  data.tar.gz: e7a506da251870a4753398365157aea3c1948944182eea55cc781478a6aabf51ffb0cae2d2b1dc4b761946da0e940ea3d8659d277a768541d70d0d8858c12a73
+  metadata.gz: d6e19f01bc6b7fdb3168fbe5ad272182e590e685c1a12d3fa97c7064791f825efdd72cca971aa385af6ecb443a6e28e3052168d7bf6aaccf458ae4cfaac37cd1
+  data.tar.gz: 66e2a1c85f1667ac98451fc943b25e9697ed7c227854ccf0e31ce214a68b56918a6f68cc1a21688a29b4bb360cafcc2711e12e493313c63c9e96908baff07e3e

data/lib/parsanol/atoms/base.rb

@@ -35,20 +35,15 @@ module Parsanol
         # Compiled-VM fast path: String input, full-consumption semantics.
         # On failure (or unsupported grammar / budget exhaustion) fall
         # through to the interpreter, which also produces the exact
-        # cause-tree diagnostics.
+        # cause-tree diagnostics. run_for memoizes heavy-backtracking
+        # grammars transparently.
         if must_consume_all && source.is_a?(String) &&
             (program = VM.program_for(self))
-          result = VM.run(program, source, true)
+          result = VM.run_for(self, program, source, true)
           if result == VM::BAIL
             # Internal bail: fall back to the interpreter and skip the VM
             # for this grammar from now on.
             VM.disable_for!(self)
-          elsif result.first == :heavy
-            # Succeeded but burned >1/8 of the step budget —
-            # heavy-backtracking grammar; the memoizing interpreter is
-            # the better engine from now on.
-            VM.disable_for!(self)
-            return finalize_result(result[1])
           elsif result.first
             return finalize_result(result[1])
           end
@@ -185,6 +180,11 @@ module Parsanol
       # Alias for ok (legacy compatibility)
       alias succ ok
 
+      # Finalizes result by flattening.
+      def finalize_result(value)
+        flatten(value)
+      end
+
       private
 
       # Converts raw input to Source if needed.
@@ -223,11 +223,6 @@ module Parsanol
 
         cause.raise
       end
-
-      # Finalizes result by flattening.
-      def finalize_result(value)
-        flatten(value)
-      end
     end
   end
 end

data/lib/parsanol/native.rb

@@ -194,7 +194,20 @@ module Parsanol
       # pre-serialized JSON grammars the native message is wrapped in a
       # Parsanol::ParseFailed directly.
       def raise_native_parse_error(error, grammar, input)
-        return grammar.parse(input) if grammar.respond_to?(:parse)
+        if grammar.respond_to?(:parse)
+          # One interpreter pass with the error reporter attached covers
+          # both jobs: a success means the native backend could not
+          # express the grammar (recover the tree), a failure raises the
+          # parslet-compatible cause tree. The native backend has
+          # already failed, so a separate plain attempt first would be a
+          # wasted parse.
+          reporter = Parsanol::ErrorReporter::Tree.new
+          source = Parsanol::Source.new(input)
+          success, value = grammar.run_with_context(source, reporter, true)
+          return grammar.finalize_result(value) if success
+
+          value.raise
+        end
 
         source = Parsanol::Source.new(input)
         cause = Parsanol::Cause.new(error.message, source, source.bytepos)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.15"
+  VERSION = "1.3.16"
 end

data/lib/parsanol/vm.rb

@@ -71,9 +71,18 @@ module Parsanol
     # and may skip the VM for this grammar afterwards. A clean
     # [false, nil] means the input genuinely does not parse.
     BAIL = Object.new.freeze
+    # Step budget exhausted on the naive pass — internal to #run, which
+    # retries once with packrat memoization before giving up.
+    BUDGET = Object.new.freeze
+    # Memo marker for a subroutine currently being evaluated at a given
+    # position. Re-entry (left recursion) fails that path instead of
+    # looping forever.
+    PENDING = Object.new.freeze
 
     STEP_BUDGET_FACTOR = 200
     STEP_BUDGET_FIXED = 10_000
+    MEMO_BUDGET_FACTOR = 4_096
+    MEMO_BUDGET_CELLS_FACTOR = 16
 
     class << self
       @programs = {}
@@ -108,13 +117,27 @@ module Parsanol
 
       def clear_program_cache
         @programs&.clear
+        @heavy&.clear
       end
 
       # Compiles the grammar rooted at +atom+. Returns the flat program
       # Array, or nil when the grammar uses unsupported atoms.
       def compile(atom)
+        program = compile_with_inline(atom, true)
+        # Inlining duplicates every non-recursive rule body at each
+        # reference site; grammars with many cross-referencing rules
+        # explode past the program cap even though the atom tree is
+        # modest. Recompiling with inlining off (each rule a CALL/RET
+        # subroutine) keeps the program O(atoms) at a small per-rule
+        # dispatch cost — far better than refusing the grammar outright.
+        return program unless program == :oversize
+
+        compile_with_inline(atom, false)
+      end
+
+      def compile_with_inline(atom, inline)
         root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
-        compiler = Compiler.new
+        compiler = Compiler.new(inline: inline)
         return nil unless compiler.compile_atom(root, true)
 
         # HALT terminates the MAIN program; subroutines follow it so the
@@ -128,8 +151,58 @@ module Parsanol
       # Executes a compiled program. Returns [true, value] on success;
       # [false, nil] on failure or budget exhaustion — callers fall back
       # to the interpreter, which reproduces exact diagnostics.
+      #
+      # Heavy-backtracking grammars get a second chance: the naive pass
+      # runs without memoization (zero overhead on the happy path), and
+      # only if it blows the step budget does a memoized pass run. The
+      # memo converts repeated rule+position work into single entries, so
+      # grammars that the interpreter needed (slow) memoization for now
+      # run at VM speed.
       def run(program, input, consume_all)
-        Executor.new(program, input, consume_all).execute
+        executor = Executor.new(program, input, consume_all)
+        result = executor.execute
+        if result.equal?(BUDGET)
+          executor = Executor.new(program, input, consume_all, memoize: true)
+          result = executor.execute
+          result = BAIL if result.equal?(BUDGET)
+        end
+        result
+      end
+
+      # Grammar-aware entry used by Base#parse. Grammars that have shown
+      # heavy-backtracking behavior (budget bust, or a success that
+      # burned the step-density threshold) memoize from the first
+      # instruction on subsequent parses, skipping the doomed naive
+      # pass; a heavy success is retried memoized in the same call so
+      # the VM keeps the grammar instead of surrendering it to the
+      # interpreter.
+      def run_for(atom, program, input, consume_all)
+        root = atom.is_a?(Parsanol::Parser) ? atom.root : atom
+        id = root.object_id
+        heavy = (@heavy ||= {})
+        if heavy[id]
+          result = Executor.new(program, input, consume_all,
+                                memoize: true).execute
+          return BAIL if result.equal?(BUDGET)
+
+          return result
+        end
+
+        result = Executor.new(program, input, consume_all).execute
+        if result.equal?(BUDGET) ||
+            (result.is_a?(Array) && result.first == :heavy)
+          heavy[id] = true
+          memo_result = Executor.new(program, input, consume_all,
+                                     memoize: true).execute
+          # A heavy naive success already answered; the memo pass only
+          # proves the grammar stays VM-viable (BUDGET would bail to the
+          # interpreter). When the naive pass never answered, the memo
+          # pass's outcome is the result.
+          return BAIL if memo_result.equal?(BUDGET)
+
+          result = memo_result if result.equal?(BUDGET)
+        end
+        result
       end
 
       # Converts the VM value tree into the interpreter's value tree.
@@ -159,8 +232,9 @@ module Parsanol
     # as subroutines, keyed by [body object_id, consume_all] so spine
     # sites (consume_all=true) and inner sites stay semantically distinct.
     class Compiler
-      def initialize
+      def initialize(inline: true)
         @ops = []
+        @inline = inline
         @subs = {}      # [obj_id, flag] => pc or :pending
         @pending = {}   # [obj_id, flag] => [[call_idx, body], ...]
       end
@@ -168,7 +242,7 @@ module Parsanol
       attr_reader :ops
 
       def to_program
-        return nil if @ops.size > MAX_PROGRAM
+        return :oversize if @ops.size > MAX_PROGRAM
 
         @ops.flatten!(1)
       end
@@ -244,7 +318,21 @@ module Parsanol
 
           emit(DROP)
           self
-          # Dynamic, Capture, Scope, Cut, Custom, Infix, unknown
+        when Parsanol::Atoms::Scope
+          # Scope only affects capture state; the result tree is the
+          # inner atom's tree unchanged. The block is pure DSL evaluated
+          # once at compile time. Any Capture/Dynamic inside still fails
+          # compilation on its own, so passthrough cannot diverge.
+          inner = begin
+            atom.block.call
+          rescue StandardError
+            nil
+          end
+          return nil if inner.nil?
+          return nil unless compile_atom(inner, consume_all)
+
+          self
+          # Dynamic, Capture, Cut, Custom, Infix, unknown
         end
       end
 
@@ -401,7 +489,7 @@ module Parsanol
         # round trip entirely. A body currently being compiled (directly
         # or indirectly) is recursive and must stay a subroutine.
         # rubocop:disable Lint/HashCompareByIdentity -- object_id keys; would otherwise pin every atom alive
-        unless (@compiling ||= {})[body.object_id]
+        if @inline && !(@compiling ||= {})[body.object_id]
           @compiling[body.object_id] = true
           # rubocop:enable Lint/HashCompareByIdentity
           result = compile_atom(body, consume_all)
@@ -456,10 +544,11 @@ module Parsanol
     # (with its adaptive memoization) is the better engine; #run reports
     # this via :heavy so Base#parse can skip the VM for the grammar.
     class Executor
-      def initialize(program, input, consume_all)
+      def initialize(program, input, consume_all, memoize: false)
         @ops = program
         @input = input
         @consume_all = consume_all
+        @memoize = memoize
       end
 
       def execute # rubocop:disable Metrics/MethodLength, Metrics/BlockLength, Metrics/BlockNesting -- single dispatch loop; hot path
@@ -471,7 +560,15 @@ module Parsanol
         # interpreter's Source#matches? semantics exactly.
         scanner = StringScanner.new(input)
         n = bytes.size
-        budget = (STEP_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+        # The memoized pass is polynomial (bounded by distinct rule ×
+        # position pairs); give it room proportional to grammar + input so
+        # packrat coverage of failing inputs isn't cut short.
+        budget = if @memoize
+                   (MEMO_BUDGET_CELLS_FACTOR * ops.size) +
+                     (MEMO_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+                 else
+                   (STEP_BUDGET_FACTOR * n) + STEP_BUDGET_FIXED
+                 end
         steps = 0
 
         pc = 0
@@ -481,13 +578,24 @@ module Parsanol
         frames = []
         calls = []
 
+        memo = @memoize ? {} : nil
+        memo_stack = []
+        trace = ENV["VM_TRACE"] ? [] : nil
+
         # rubocop:disable-next Metrics/BlockLength -- the dispatch loop IS execute
         loop do
           steps += 1
-          return BAIL if steps > budget
+          if steps > budget
+            if trace
+              warn "BUDGET at steps=#{steps} pc=#{pc} pos=#{pos} rstack=#{rstack.size} bt=#{bt.size} calls=#{calls.size}"
+              warn "trace tail: #{trace.last(30).inspect}"
+            end
+            return BUDGET
+          end
+          trace << [pc, pos] if trace
 
           if pc == FAIL
-            pc, pos = unwind(bt, rstack, frames, calls)
+            pc, pos = unwind(bt, rstack, frames, calls, memo, memo_stack)
             return [false, nil] if pc == :fail
             return BAIL if pc == :bail
 
@@ -579,11 +687,45 @@ module Parsanol
             rstack << VM::NamedValue.new(ops[pc + 1], rstack.pop)
             pc += 4
           when CALL
+            sub = ops[pc + 1]
+            if memo
+              table = (memo[sub] ||= {})
+              hit = table[pos]
+              if hit
+                if hit.equal?(PENDING) || hit.equal?(:fail)
+                  # :fail — the body already failed at this position;
+                  # replay the failure without re-running it.
+                  # PENDING — re-entry at the same rule+position (left
+                  # recursion); the interpreter loops forever here, so
+                  # failing this path keeps the VM bounded.
+                  pc = FAIL
+                  next
+                end
+                # Memo hit: replay the subroutine's stack effect without
+                # executing it. Values are immutable (packed spans,
+                # NamedValue, frozen-shape Arrays), so sharing is safe.
+                rstack.concat(hit[1])
+                pos = hit[0]
+                pc += 4
+                next
+              end
+
+              table[pos] = PENDING
+              # Lockstep with calls: unwind rolls both back by clen.
+              memo_stack << sub << pos << rstack.size
+            end
             calls << (pc + 4)
-            pc = ops[pc + 1]
+            pc = sub
           when RET
             pc = calls.pop
             return BAIL if pc.nil?
+
+            if memo
+              mlen = memo_stack.pop
+              mpos = memo_stack.pop
+              msub = memo_stack.pop
+              memo[msub][mpos] = [pos, rstack[mlen..]]
+            end
           when LOOK_POS
             bt << ops[pc + 1] << pos << rstack.size << frames.size << calls.size << K_LOOK_FAIL
             pc += 4
@@ -735,6 +877,8 @@ module Parsanol
                   values << [:maybe]
                 end
               when RUN_RE
+                elem_start = pos
+                count = 0
                 loop do
                   b = bytes[pos]
                   m = if kb && b && b < 128
@@ -746,12 +890,19 @@ module Parsanol
                       end
                   break unless m
 
+                  count += 1
                   w = CHAR_WIDTH[b]
                   w = char_width_slow(input, pos) if w.zero?
                   values << ((pos << 20) | w)
                   pos += w
                 end
+                if count < kc
+                  pos = elem_start
+                  failed = true
+                end
               when RUN_STR
+                elem_start = pos
+                count = 0
                 loop do
                   i = 0
                   ln = kb
@@ -765,9 +916,14 @@ module Parsanol
                   end
                   break unless m
 
+                  count += 1
                   values << ((pos << 20) | ln)
                   pos += ln
                 end
+                if count < kc
+                  pos = elem_start
+                  failed = true
+                end
               end
               break if failed
 
@@ -878,6 +1034,10 @@ module Parsanol
             value = VM.materialize(rstack[0], input)
             # More than ~100 steps per input byte means heavy
             # backtracking; the memoizing interpreter wins there.
+            if @memoize
+              return [true, value]
+            end
+
             return steps > (n << 9) + 1000 ? [:heavy, value] : [true, value]
           else
             return BAIL
@@ -890,7 +1050,7 @@ module Parsanol
       # Unwind one backtrack entry. Returns [pc, pos] or :fail. Repetition
       # entries with count >= min complete their tagged array and jump to
       # the exit continuation with the last iteration end position.
-      def unwind(bt, rstack, frames, calls) # rubocop:disable Naming/MethodParameterName -- stack register names
+      def unwind(bt, rstack, frames, calls, memo, memo_stack) # rubocop:disable Naming/MethodParameterName -- stack register names
         return :fail if bt.empty?
 
         # Everything above this entry belongs to constructs being
@@ -910,6 +1070,24 @@ module Parsanol
 
         rstack.slice!(rlen..) if rstack.size > rlen
         calls.slice!(clen..) if calls.size > clen
+        if memo
+          # memo_stack carries three slots per live call, so the frames of
+          # every call abandoned by this unwind are the tail above 3*clen.
+          # Every abandoned frame is a call whose body failed — cache that
+          # failure too (full packrat memoization), so later explorations
+          # that reach the same rule+position fail immediately instead of
+          # re-running the body. Grammar inputs that don't parse explore
+          # every alternative; without failure entries those explorations
+          # repeat exponentially and blow the step budget.
+          floor = clen * 3
+          while memo_stack.size > floor
+            memo_stack.pop
+            mpos = memo_stack.pop
+            msub = memo_stack.pop
+            table = memo[msub]
+            table[mpos] = :fail if table
+          end
+        end
 
         case kind
         when K_ALT
@@ -930,7 +1108,7 @@ module Parsanol
             check_full = @ops[epc + 2]
             if check_full == 1 && end_pos != @input.bytesize
               frames.slice!(cbase..)
-              unwind(bt, rstack, frames, calls)
+              unwind(bt, rstack, frames, calls, memo, memo_stack)
             else
               values = rstack.pop(rstack.size - rbase)
               values.unshift(tag)
@@ -940,11 +1118,11 @@ module Parsanol
             end
           else
             frames.slice!(cbase..)
-            unwind(bt, rstack, frames, calls)
+            unwind(bt, rstack, frames, calls, memo, memo_stack)
           end
         when K_LOOK_FAIL
           frames.slice!(cbase..) if frames.size > cbase
-          unwind(bt, rstack, frames, calls)
+          unwind(bt, rstack, frames, calls, memo, memo_stack)
         when K_NEG
           # Negative lookahead body failed -> lookahead succeeds.
           frames.slice!(cbase..) if frames.size > cbase

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.15
+  version: 1.3.16
 platform: aarch64-linux
 authors:
 - Ribose Inc.