parsanol 1.3.115 → 1.3.116

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 116a1ab6a5ed95d4b3998c1d1d0b4860ed7dabafb5464c8aa8ff7d8602d9c221
-  data.tar.gz: 0f4799930aa443744b0ed3367fc139ab18eb565aca24a640a642d37b80596736
+  metadata.gz: b6be4ab27d5d8c86b4f214b75cefb0bb16a3de4eb9e55402d93ca807fae280aa
+  data.tar.gz: cf56a4e2921063811e55a4016342cce471acf24138825ebe24673f5563cf70ea
 SHA512:
-  metadata.gz: df5ecd79facb4f09fcbbd5328ce3a4d2713fb230322a1f6ff4fccfd68fc6b12cbb6fa5db573df86035e3ce85c1626bed58b75c494a38701305456d35c34f6502
-  data.tar.gz: c1f70bd90dd3ba48126dbb010d9ba89013b39cab654284224afe5031d7713caae84cc3c26b6b454f508b6fa7083504a52bc49dc1c18651a9c381042a94588b71
+  metadata.gz: 13957312679b48cf55f8455e8a6311c785915dd95e9c940d974f487bc814ec07de54fe30f860912bfc9b42b7b51645110b95e6d52ccba266951d3d9523d864f7
+  data.tar.gz: 852e7731ce29b534bd383010655cb6380c9bac8840c69204904d1e17de089e5db578c13ad3c65707aa8d686ebd5ffc50ea526035256a474c8cdf7ea555cb1838

data/Cargo.lock

@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.19.10"
+version = "0.19.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19a06284aa6f46b116ad287275b3febfbd5f36b0f85b130f492d8244afea8c13"
+checksum = "2d8c6e0a21c9ee8cc6e661e4109a70d1c8f4f62c963eb12230b8d7035ebb63ee"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -326,9 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.19.10"
+version = "0.19.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd454669ba1439d5c4f1bb3cd71861cda751586a3ddf2521383f8fbbb4b19049"
+checksum = "2662678735acff3124dc2514d4792cc4a82cfaff50ed3b0a88c2a5662ecb9818"
 dependencies = [
  "proc-macro2",
  "quote",

data/ext/parsanol_native/Cargo.toml

@@ -27,12 +27,8 @@ rb-sys = { version = "0.9.124", features = ["global-allocator"] }
 # magnus for Ruby bindings (from crates.io, patched locally for Ruby 4.0 support)
 magnus = { version = "0.9" }
 
-# parsanol parser library (crates.io; 0.12 carries the rs#174 native
-# backtracking fix: dynamic-dependent memoization + outcome cache)
-# parsanol parser library (crates.io; 0.17.0 carries the LookSpec regex
-# lookbehind (rs#163 gap 1) and the widened opaque writeback: the
-# lazy-continuation fix, parsanol-ruby#160)
-parsanol = { version = "0.19.1", features = ["ruby"] }
+# Native legacy compatibility and compact recursive parser results.
+parsanol = { version = "0.19.12", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/ext/parsanol_native/extconf.rb

@@ -30,6 +30,15 @@ create_rust_makefile("parsanol/parsanol_native") do |r|
   # Create debug builds in dev, release in production
   r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :dev).to_sym
 
+  # Persist RubyInstaller's headers into Make's later Cargo subprocess.
+  if RUBY_PLATFORM.include?("mingw") && ENV.key?("RI_DEVKIT") && ENV.key?("MSYSTEM_PREFIX")
+    sysroot = ENV.fetch("RI_DEVKIT").tr("\\", "/") + ENV.fetch("MSYSTEM_PREFIX")
+    r.env["BINDGEN_EXTRA_CLANG_ARGS"] = ENV.fetch("BINDGEN_EXTRA_CLANG_ARGS") do
+      "--target=x86_64-pc-windows-gnu --sysroot=#{sysroot} " \
+        "-D__AVX512VLFP16INTRIN_H -D__AVX512FP16INTRIN_H"
+    end
+  end
+
   # cargo rustc builds dependency cdylibs verbatim, and parsanol ships
   # a cdylib crate-type (the ffi tier). On macOS every dylib in the
   # build resolves ruby symbols from the host process at load time, so

data/lib/parsanol/atoms/alternative.rb

@@ -117,7 +117,9 @@ module Parsanol
         # The reporting pass (the re-parse after a failure) must try every
         # branch so the failure cause tree stays complete; the literal index
         # only prunes the fast non-reporting pass.
-        unless context.reporting?
+        # Legacy memo answers depend on attempted branches, including a
+        # prefix success in a branch that later fails its literal suffix.
+        unless context.reporting? || context.parslet_compatible?
           indexed = candidate_indexes(source)
           if indexed
             return try_selected(indexed, options, source, context, consume_all)

data/lib/parsanol/atoms/base.rb

@@ -27,17 +27,27 @@ module Parsanol
       # @param options [Hash] parsing options
       # @option options [Parsanol::ErrorReporter] :reporter error collector
       # @option options [Boolean] :prefix allow partial parse (default: false)
+      # @option options [Symbol] :compatibility :strict (default), or :parslet
+      #   for legacy consume-all memo decisions on the Ruby interpreter
       # @return [Object] the parsed result
       # @raise [Parsanol::ParseFailed] on parse failure
       def parse(source, options = {})
         must_consume_all = !options[:prefix]
+        compatibility = options.fetch(:compatibility, :strict)
+        unless %i[strict parslet].include?(compatibility)
+          raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
+        end
+
+        context_options = compatibility == :strict ? {} : { compatibility: compatibility }
 
         # Compiled-VM fast path: String input, full-consumption semantics.
         # On failure (or unsupported grammar / budget exhaustion) fall
         # through to the interpreter, which also produces the exact
         # cause-tree diagnostics. run_for memoizes heavy-backtracking
         # grammars transparently.
-        if must_consume_all && source.is_a?(String) &&
+        # The VM implements strict memo semantics. The opt-in legacy policy
+        # is currently supported by the Ruby interpreter only.
+        if compatibility == :strict && must_consume_all && source.is_a?(String) &&
             (program = VM.program_for(self))
           begin
             result = VM.run_for(self, program, source, true)
@@ -62,12 +72,12 @@ module Parsanol
         input = normalize_input(source)
 
         # Initial parse attempt (no error collection)
-        success, value = run_with_context(input, nil, must_consume_all)
+        success, value = run_with_context(input, nil, must_consume_all, **context_options)
         return finalize_result(value) if success
 
         # Reparse with error reporting for diagnostics
         report_detailed_error(input, must_consume_all, options[:reporter],
-                              value)
+                              value, **context_options)
       end
 
       # Creates a new parsing context and executes the atom.
@@ -76,10 +86,11 @@ module Parsanol
       # @param reporter [Object, nil] error reporter
       # @param consume_all [Boolean] require complete consumption
       # @return [Array(Boolean, Object)] outcome tuple
-      def run_with_context(input, reporter, consume_all)
+      def run_with_context(input, reporter, consume_all, compatibility: :strict)
         parser_class = detect_parser_class
         context = Parsanol::Atoms::Context.new(reporter,
-                                               parser_class: parser_class)
+                                               parser_class: parser_class,
+                                               compatibility: compatibility)
         apply(input, context, consume_all)
       end
 
@@ -242,10 +253,12 @@ module Parsanol
       # consumer's file:line:col degrades to "line 1 char 1" when the
       # interpreter raises the outermost sequence instead. Explicit
       # reporters (Tree for full cause trees) are honored unchanged.
-      def report_detailed_error(input, consume_all, reporter, _initial_error)
+      def report_detailed_error(input, consume_all, reporter, _initial_error, compatibility: :strict)
         input.bytepos = 0
-        error_reporter = reporter || Parsanol::ErrorReporter::Deepest.new
-        success, cause = run_with_context(input, error_reporter, consume_all)
+        error_reporter = reporter ||
+          (compatibility == :parslet ? Parsanol::ErrorReporter::Tree.new : Parsanol::ErrorReporter::Deepest.new)
+        context_options = compatibility == :strict ? {} : { compatibility: compatibility }
+        success, cause = run_with_context(input, error_reporter, consume_all, **context_options)
 
         # Second parse should also fail
         raise "Invariant violation: parse succeeded during error reporting" if success

data/lib/parsanol/atoms/context.rb

@@ -44,11 +44,24 @@ module Parsanol
       # @param interval_cache: [Boolean] enable GPeg-style interval caching
       # @param adaptive_cache_threshold: [Integer, nil] minimum input size for caching
       # @param parser_class: [Class, nil] parser class for threshold selection
+      # @param compatibility: [Symbol] :strict or opt-in :parslet. Legacy
+      #   decisions use eager, parse-lifetime memo entries; mutable capture
+      #   and dynamic evaluations still retain their cache barriers.
       #
       def initialize(error_reporter = Parsanol::ErrorReporter::Tree.new,
                      interval_cache: false,
                      adaptive_cache_threshold: nil,
-                     parser_class: nil)
+                     parser_class: nil,
+                     compatibility: :strict)
+        unless %i[strict parslet].include?(compatibility)
+          raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
+        end
+
+        @parslet_compatibility = compatibility == :parslet
+        if @parslet_compatibility && interval_cache
+          raise NotImplementedError, "Parslet compatibility does not yet support interval caching"
+        end
+
         # Core memoization cache: position -> { atom_id -> [result, advance] }
         # Auto-vivifying sub-hashes: a probe-phase position can reach the
         # active-path lookup before any entry exists at that position
@@ -84,11 +97,6 @@ module Parsanol
         @array_pool = Parsanol::Pools::ArrayPool.new(size: 10_000, preallocate: false)
         @buffer_pool = Parsanol::Pools::BufferPool.new(pool_size: 100)
 
-        # Selective memoization tracking
-        @hit_stats = Hash.new(0)
-        @miss_stats = Hash.new(0)
-        @min_hits_for_cache = 2
-
         # Optional GPeg-style interval caching
         @use_intervals = interval_cache
         if @use_intervals
@@ -117,7 +125,7 @@ module Parsanol
 
         @adaptive_threshold = threshold
         @input_len = nil
-        @caching_active = false
+        @caching_active = @parslet_compatibility
         @backtrack_events = 0
 
         # Monotonic counter of cache-unsafe events (dynamic-atom evaluations
@@ -132,6 +140,10 @@ module Parsanol
       # @return [Integer] monotonic count of cache-unsafe events so far
       attr_reader :cache_unsafe_events
 
+      def parslet_compatible?
+        @parslet_compatibility
+      end
+
       # Records that mutable parse state was read or written (dynamic atom
       # evaluated, capture stored). Called by atoms; results computed across
       # such events are excluded from memoization.
@@ -152,7 +164,11 @@ module Parsanol
       #
       def try_with_cache(atom, src, must_consume_all)
         # Skip caching for atoms that don't benefit from it
-        return atom.try(src, self, must_consume_all) unless atom.cached?
+        legacy_pure = @parslet_compatibility &&
+          (atom.instance_of?(Parsanol::Atoms::Entity) ||
+           atom.instance_of?(Parsanol::Atoms::Str) || atom.instance_of?(Parsanol::Atoms::Re))
+        cacheable = atom.cached? || legacy_pure
+        return atom.try(src, self, must_consume_all) unless cacheable
 
         # Use interval-based caching if enabled
         return try_with_interval(atom, src, must_consume_all) if @use_intervals
@@ -171,7 +187,9 @@ module Parsanol
           @furthest_pos = pos
           @evict_counter += 1
 
-          if @evict_counter >= @evict_interval
+          # Legacy prefix decisions remain observable on later backtracking;
+          # retain them for this parse rather than silently re-evaluating.
+          if !@parslet_compatibility && @evict_counter >= @evict_interval
             @evict_counter = 0
             cutoff = pos - @evict_threshold
             @memo.delete_if { |p, _| p < cutoff }
@@ -182,26 +200,21 @@ module Parsanol
         # prefix-success fallback for ordered choice)
         cached_key = cached_entry_key(@memo[pos], atom, must_consume_all)
         if cached_key
-          @hit_stats[cached_key] += 1
           outcome, delta = @memo[pos][cached_key]
           src.bytepos = pos + delta
           return outcome
         end
 
         # Cache miss - execute and store
-        @miss_stats[key] += 1
         unsafe_before = @cache_unsafe_events
         outcome = atom.try(src, self, must_consume_all)
         delta = src.bytepos - pos
 
-        # Never store results spanning cache-unsafe events (dynamic
-        # blocks, capture writes); keep the beneficial-attempts
-        # heuristic to avoid one-shot memo pollution.
-        if storable_outcome?(key, outcome, unsafe_before)
-          attempts = @hit_stats[key] + @miss_stats[key]
-          if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
-            (@memo[pos] ||= {})[key] = [outcome, delta]
-          end
+        # Once backtracking activates memoization, retain every safe answer.
+        # An atom-wide attempts limit suppresses its entries at later positions
+        # and can leave recursive alternatives with exponential repeated work.
+        if @cache_unsafe_events == unsafe_before
+          (@memo[pos] ||= {})[key] = [outcome, delta]
         end
 
         outcome
@@ -217,23 +230,21 @@ module Parsanol
       def try_with_interval(atom, src, must_consume_all)
         pos = src.bytepos
         key = scoped_cache_key(atom, must_consume_all)
-        cached_key, cached = cached_interval_entry(atom, pos,
-                                                   must_consume_all)
+        _cached_key, cached = cached_interval_entry(atom, pos,
+                                                    must_consume_all)
 
         if cached
-          @hit_stats[cached_key] += 1
           outcome, delta = cached
           src.bytepos = pos + delta
           return outcome
         end
 
-        @miss_stats[key] += 1
         unsafe_before = @cache_unsafe_events
         outcome = atom.try(src, self, must_consume_all)
         delta = src.bytepos - pos
         end_pos = pos + delta
 
-        if storable_outcome?(key, outcome, unsafe_before)
+        if @cache_unsafe_events == unsafe_before
           @interval_trees[key].insert(pos, end_pos, [outcome, delta])
         end
 
@@ -533,6 +544,8 @@ module Parsanol
       end
 
       def scoped_cache_key(atom, must_consume_all)
+        return shared_cache_key(atom) if @parslet_compatibility
+
         must_consume_all ? strict_cache_key(atom) : shared_cache_key(atom)
       end
 
@@ -554,17 +567,6 @@ module Parsanol
         !must_consume_all && share_prefix_success_cache?(atom)
       end
 
-      # An outcome may be memoized only when no cache-unsafe event (dynamic
-      # evaluation, capture write) happened while computing it, and the
-      # attempts heuristic deems the entry worthwhile.
-      def storable_outcome?(key, outcome, unsafe_before)
-        return false unless @cache_unsafe_events == unsafe_before
-
-        outcome.first ||
-          @hit_stats[key] + @miss_stats[key] <= @min_hits_for_cache ||
-          @hit_stats[key].positive?
-      end
-
       def successful_prefix_entry?(entry)
         entry && entry[0].is_a?(Array) && entry[0].first == true
       end

data/lib/parsanol/atoms/repetition.rb

@@ -74,6 +74,8 @@ module Parsanol
       # @param consume_all [Boolean] require full consumption
       # @return [Array(Boolean, Object)] result
       def try(source, context, consume_all)
+        return try_compatible(source, context, consume_all) if context.parslet_compatible?
+
         # Check for tree memoization support
         if context.use_tree_memoization?
           return with_tree_cache(source, context, consume_all)
@@ -119,6 +121,37 @@ module Parsanol
         first
       end
 
+      # Legacy repetition checks all children as prefixes. A bounded success
+      # returns before the outer consume-all guard; an exhausted optional
+      # failure checks consume-all inside #try. Their raw memo answers differ.
+      def try_compatible(source, context, consume_all)
+        start_pos = source.bytepos
+        result = [@result_tag]
+        count = 0
+        failure = nil
+        loop do
+          previous = source.bytepos
+          success, value = @parslet.apply(source, context, false)
+          unless success
+            failure = value
+            source.bytepos = previous
+            break
+          end
+
+          count += 1
+          result << value
+          return ok(result) if @max && count >= @max
+          # Retain the current engine's protection against empty loops.
+          break if source.bytepos == previous
+        end
+        return context.err_at(self, source, min_error, start_pos, [failure]) if count < @min
+        if consume_all && source.chars_left.positive?
+          return context.err(self, source, @extra_error, [failure])
+        end
+
+        ok(result)
+      end
+
       # Optional match (0 or 1)
       def try_maybe(source, context, _consume_all)
         success, value = @parslet.apply(source, context, false)

data/lib/parsanol/native/parser.rb

@@ -57,9 +57,9 @@ module Parsanol
         #
         # @param grammar [Parsanol::Atoms::Base] Ruby grammar or JSON string
         # @param input [String] Input string to parse
-        def parse(grammar, input)
+        def parse(grammar, input, **)
           # Delegate to Parsanol::Native.parse for consistency
-          Parsanol::Native.parse(grammar, input)
+          Parsanol::Native.parse(grammar, input, **)
         end
 
         # Parse input with a Ruby grammar, returning the parslet-shaped
@@ -84,8 +84,13 @@ module Parsanol
         # Resolve a Rust-side handle for the grammar, registering it once.
         # Per-call cost is the memoized structure-hash lookup plus one Hash
         # access — no JSON marshal, no re-hash inside Rust.
-        def grammar_handle(root_atom)
-          cache_key = grammar_cache_key(root_atom)
+        def grammar_handle(root_atom, compatibility: :strict)
+          root_atom = root_atom.root if root_atom.is_a?(::Parsanol::Parser)
+          legacy = compatibility == :parslet
+          # Legacy decisions depend on the exact shared-node graph. The
+          # strict structure hash deliberately omits repeated-node targets.
+          # Retaining the root in this bounded key also prevents id recycling.
+          cache_key = legacy ? [:parslet, root_atom] : grammar_cache_key(root_atom)
           handle = HANDLE_CACHE[cache_key]
           if handle
             # Refresh recency: reinsertion moves the entry to the end,
@@ -94,7 +99,8 @@ module Parsanol
             return handle
           end
 
-          handle = Native._register_grammar(grammar_json(root_atom))
+          json = legacy ? GrammarSerializer.serialize(root_atom) : grammar_json(root_atom)
+          handle = Native._register_grammar(json)
           HANDLE_CACHE[cache_key] = handle
           trim_caches
           handle

data/lib/parsanol/native.rb

@@ -58,9 +58,24 @@ module Parsanol
       #   result = Parsanol::Native.parse(str('hello').as(:greeting), "hello\nworld")
       #   result[:greeting].line_and_column  # => [1, 1]
       #
-      def parse(grammar, input)
+      def parse(grammar, input, compatibility: :strict)
+        unless %i[strict parslet].include?(compatibility)
+          raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
+        end
         raise LoadError, "Native parser not available" unless available?
 
+        if compatibility == :parslet
+          unless Parser.extension_loaded? && respond_to?(:_parse_handle_with_compatibility)
+            raise NotImplementedError,
+                  "Native Parslet compatibility requires an extension with _parse_handle_with_compatibility"
+          end
+          if grammar.is_a?(String)
+            raise NotImplementedError, "Native Parslet compatibility requires grammar atoms, not JSON"
+          end
+
+          return parse_compatible_atom_grammar(grammar, input)
+        end
+
         # ffi-gem tier (JRuby/TruffleRuby/no-binary MRI): same contract,
         # batch-decoded results, single reporter-pass error fallback.
         return Ffi.parse(grammar, input) unless Parser.extension_loaded?
@@ -366,6 +381,35 @@ module Parsanol
         end
       end
 
+      # Compatibility failures are authoritative native refusals. Unlike the
+      # legacy strict helper, this path never reparses or accepts in Ruby.
+      def parse_compatible_atom_grammar(grammar, input)
+        handle = Parser.grammar_handle(grammar, compatibility: :parslet)
+        raw = begin
+          _parse_handle_with_compatibility(handle, input, "parslet")
+        rescue ArgumentError
+          Parser.invalidate_handle(handle)
+          begin
+            _parse_handle_with_compatibility(
+              Parser.grammar_handle(grammar, compatibility: :parslet), input, "parslet"
+            )
+          rescue RuntimeError => e
+            raise_compatible_parse_error(e, input)
+          end
+        rescue RuntimeError => e
+          raise_compatible_parse_error(e, input)
+        end
+        BatchDecoder.decode_and_flatten(raw, input, Parsanol::Slice)
+      end
+
+      def raise_compatible_parse_error(error, input)
+        message = error.message
+        position = message[NATIVE_POS_MARKER, 1].to_i
+        cause = Parsanol::Cause.new(message.sub(NATIVE_POS_MARKER, ""),
+                                    Parsanol::Source.new(input), position)
+        raise Parsanol::ParseFailed.new(cause.to_s, cause), cause: nil
+      end
+
       # Reference-walker path (TODO.sota/8 B1): the plain recursive
       # relation — no VM, no memo tables, no fused scans, no terminal
       # fast paths. Deliberately slow; the cross-engine differential

data/lib/parsanol/parser.rb

@@ -176,6 +176,9 @@ module Parsanol
     # Options:
     # - :reporter - Custom error reporter instance
     # - :prefix - Allow partial matching (default: false)
+    # - :compatibility - :strict (default), or :parslet for legacy memo
+    #   decisions. Auto mode selects native when available; an older native
+    #   extension without the compatibility entrypoint raises unsupported.
     #
     # @return [Hash, Array, Parsanol::Slice] parsed result with position info
     # @raise [Parsanol::ParseFailed] when parsing fails
@@ -200,8 +203,22 @@ module Parsanol
       # parse(input, {mode: :ruby, ...}) is the same call shape a subclass
       # forwarding options via super produces; :mode must be honored
       # wherever it appears, or those callers silently get the native path.
-      mode = opts.delete(:mode) ||
-        (Parsanol::Native.available? && native_expressible? ? :native : :ruby)
+      compatibility = opts.fetch(:compatibility, :strict)
+      unless %i[strict parslet].include?(compatibility)
+        raise ArgumentError, "Unknown compatibility policy: #{compatibility}"
+      end
+
+      requested_mode = opts.delete(:mode)
+      mode = requested_mode ||
+        if compatibility == :parslet
+          Parsanol::Native.available? ? :native : :ruby
+        else
+          Parsanol::Native.available? && native_expressible? ? :native : :ruby
+        end
+      if compatibility == :parslet && mode == :json
+        raise NotImplementedError, "Parslet compatibility does not yet support JSON mode"
+      end
+
       case mode
       when :ruby
         super(input, opts)
@@ -291,6 +308,14 @@ module Parsanol
     # @return [Object] parse result with Slice objects for position info
     #
     def parse_native(input, opts)
+      if opts[:compatibility] == :parslet
+        if opts[:prefix]
+          raise NotImplementedError, "Native Parslet compatibility does not yet support prefix mode"
+        end
+
+        return Parsanol::Native.parse(root, input, compatibility: :parslet)
+      end
+
       unless Parsanol::Native.available?
         Parsanol::Atoms::Base.instance_method(:parse).bind_call(self, input,
                                                                 opts)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.115"
+  VERSION = "1.3.116"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.115
+  version: 1.3.116
 platform: ruby
 authors:
 - Ribose Inc.