parsanol 1.3.47 → 1.3.48

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84b51a358f1867ed55ebfa7a9c6afbb2d442c23e93e8b44b338fd8479eb38f3c
-  data.tar.gz: 7d0e702019b1023b1ff41b36099c44b27eb8613d4e02c8b10ee60ac5501a0748
+  metadata.gz: '085fe6a0f25c1eadc37dd24b31b4c60a4b96e2224e472e8f4a300ae9b3d25ae5'
+  data.tar.gz: e770936723d693d3990cf4d051754d35f0a6d0e0f9bc6a25daef680b46f94766
 SHA512:
-  metadata.gz: 6d31070597c02776113dfdafe9749d86f2e2edcaf88700576823d10058a7c244114859f304a1c85856c29c520102dfa64d2553508a95259af1f530995e339bd6
-  data.tar.gz: d0a35ca10ec03ba72b40b1e7a37043b24065a3246bec795c19f32e3e6e023188b60402c31c775517a1d2a1576b3a392583f21f2ffb371286ee3727d498b096aa
+  metadata.gz: d33f1a5069d853dc26153b92b5994310c6b274e1ed7e966a748aa186e08c8a8519db52d264197b347cd94dbdf69bf045bfb42e80d14c1226aa11cce8b643587c
+  data.tar.gz: 5a32c726999b8870cd11913dc5ac069238790123771a0c9a11cd9dbf180dd92761b1ccc02a732661458d744281a5fde3ae192bcc4fc501153641a57fdc8a0434

data/HISTORY.txt

@@ -5,12 +5,28 @@ Breaking changes:
 * Simplified API: Single `parse()` method with lazy line/column
 * Removed deprecated methods: `parse_parslet`, `parse_parslet_with_positions`,
   `parse_with_transform`, `parse_to_objects`, `parse_raw`
+* Aligned ordered choice with Parslet's prefix-success cache semantics -
+  grammars that parse on Parslet 2.0 parse identically; some that fail there
+  now raise `ParseFailed` here too
+* Scoped memoization keys by parse mode - strict failures are no longer
+  replayed for prefix parses, fixing inputs that failed only above the
+  adaptive caching threshold
+* Choice errors enumerate at most 16 alternatives - larger choices report
+  "Expected one of N alternatives" (per-branch causes stay in the cause tree)
+* Made `Slice#hash` content-based to match `Slice#eql?` - offset no longer
+  affects Hash/Set key identity
+* Froze `Alternative#alternatives` - extend choices with `a | b` instead of
+  in-place mutation
 
 New features:
 
 * Lazy line/column computation in `Slice#line_and_column` - computed only when accessed
 * `BatchDecoder` module for efficient batch AST processing
 * Grammar accepts Ruby atoms directly - no JSON serialization step needed
+* Literal-prefix indexing for ordered choices with 16+ branches on the
+  pure-Ruby path - failure reporting still tries every branch
+* Parser-class parses memoize at all input sizes by default - thresholds are
+  performance-only since cache-unsafe results are never memoized
 
 Bug fixes:
 
@@ -19,6 +35,12 @@ Bug fixes:
   - Same inner keys → Repetition pattern (keep as array)
   - Different inner keys → Wrapper pattern (merge inner hashes)
 * Correct entity extraction for grammars with multiple declarations
+* Fixed memoization of capture-dependent results - `dynamic {}` blocks and
+  capture writes are re-evaluated on every attempt
+* Fixed repetition interval/tree memoization to actually replay cached
+  results, consistently across consume-all modes
+* Fixed `Buffer#clear!` to clear used slots, plugging a pooled-buffer
+  reference leak
 
 == Parsanol 1.2.2 (2026-03-07)
 

data/lib/parsanol/atoms/alternative.rb

@@ -11,6 +11,13 @@
 module Parsanol
   module Atoms
     class Alternative < Parsanol::Atoms::Base
+      # Literal-prefix indexing only pays off past this many alternatives
+      INDEX_THRESHOLD = 16
+      # ... and only when at least this many branches have a static literal prefix
+      INDEX_MIN_BRANCHES = 4
+      # Above this many alternatives the error message stops enumerating them
+      CHOICE_ERROR_DETAIL_LIMIT = 16
+
       # @return [Array<Parsanol::Atoms::Base>] alternative parsers
       attr_reader :alternatives
 
@@ -19,8 +26,9 @@ module Parsanol
       # @param options [Array<Parsanol::Atoms::Base>] alternatives
       def initialize(*options)
         super()
-        @alternatives = options
-        @choice_error = "Expected one of #{options.inspect}"
+        # Frozen so the lazily memoized literal index can never go stale:
+        # the index assumes this array is immutable after construction.
+        @alternatives = options.freeze
       end
 
       # Adds an alternative with flattening.
@@ -87,7 +95,7 @@ module Parsanol
         success, value2 = a2.apply(source, context, consume_all)
         return [success, value2] if success
 
-        context.err(self, source, @choice_error, [value1, value2])
+        context.err(self, source, choice_error, [value1, value2])
       end
 
       # Three-alternative fast path
@@ -101,11 +109,26 @@ module Parsanol
         success, value3 = a3.apply(source, context, consume_all)
         return [success, value3] if success
 
-        context.err(self, source, @choice_error, [value1, value2, value3])
+        context.err(self, source, choice_error, [value1, value2, value3])
       end
 
       # General case for N alternatives
       def try_many(options, source, context, consume_all)
+        # The reporting pass (the re-parse after a failure) must try every
+        # branch so the failure cause tree stays complete; the literal index
+        # only prunes the fast non-reporting pass.
+        unless context.reporting?
+          indexed = candidate_indexes(source)
+          if indexed
+            return try_selected(indexed, options, source, context, consume_all)
+          end
+        end
+
+        try_all(options, source, context, consume_all)
+      end
+
+      # Full scan over every branch (reporting pass, or no usable index)
+      def try_all(options, source, context, consume_all)
         errors = nil
 
         options.each do |alt|
@@ -116,7 +139,198 @@ module Parsanol
           errors << value
         end
 
-        context.err(self, source, @choice_error, errors)
+        context.err(self, source, choice_error, errors)
+      end
+
+      # Scan restricted to index-selected branch positions, in original order
+      def try_selected(indexes, options, source, context, consume_all)
+        errors = nil
+
+        indexes.each do |idx|
+          success, value = options[idx].apply(source, context, consume_all)
+          return [success, value] if success
+
+          errors ||= []
+          errors << value
+        end
+
+        context.err(self, source, choice_error, errors)
+      end
+
+      # Branch positions worth trying for the current input, or nil when no
+      # index applies and every branch must be scanned
+      def candidate_indexes(source)
+        index = literal_index
+        return nil unless index
+
+        indexes = index[:always].dup
+        prefixes = index[:prefixes]
+        max_prefix_bytes = index[:max_prefix_bytes]
+        current_prefix = +""
+
+        valid_prefix_preview(source.peek(max_prefix_bytes)).each_char do |char|
+          current_prefix << char
+          matches = prefixes[current_prefix]
+          indexes.concat(matches) if matches
+        end
+
+        indexes.uniq!
+        indexes.sort!
+        indexes
+      end
+
+      # Lazily builds and memoizes the literal-prefix index (nil when this
+      # choice is too small or has too few literal branches to index)
+      def literal_index
+        return nil if @alternatives.size < INDEX_THRESHOLD
+
+        # Benign lazy race: alternatives are frozen at initialization, so
+        # concurrent builds produce the same index and the last assignment wins.
+        return @literal_index if defined?(@literal_index)
+
+        @literal_index = build_literal_index
+      end
+
+      # Maps each branch's static literal prefix to its position; branches
+      # without one go to the always-tried bucket
+      def build_literal_index
+        prefixes = {}
+        always = []
+        indexed_count = 0
+        max_prefix_bytes = 0
+
+        @alternatives.each_with_index do |alt, idx|
+          prefix, = static_literal_prefix(alt)
+
+          if prefix.nil? || prefix.empty?
+            always << idx
+            next
+          end
+
+          add_to_index(prefixes, prefix, idx)
+          indexed_count += 1
+          max_prefix_bytes = [max_prefix_bytes, prefix.bytesize].max
+        end
+
+        return nil if indexed_count < INDEX_MIN_BRANCHES
+
+        {
+          prefixes: freeze_prefix_index(prefixes),
+          max_prefix_bytes: max_prefix_bytes,
+          always: always.freeze,
+        }.freeze
+      end
+
+      def add_to_index(prefixes, prefix, idx)
+        (prefixes[prefix] ||= []) << idx
+      end
+
+      def freeze_prefix_index(prefixes)
+        prefixes.transform_values(&:freeze).freeze
+      end
+
+      # Trims a byte-bounded peek down to valid encoding: the scanner position
+      # is always on a character boundary, so only a trailing character can be
+      # cut and at most a few iterations are ever needed
+      def valid_prefix_preview(preview)
+        return preview if preview.valid_encoding?
+
+        (preview.bytesize - 1).downto(1) do |bytesize|
+          trimmed = preview.byteslice(0, bytesize)
+          return trimmed if trimmed.valid_encoding?
+        end
+
+        +""
+      end
+
+      # Lazy so building a large choice never pays inspect costs up front;
+      # benign lazy race, same as literal_index
+      def choice_error
+        @choice_error ||= if @alternatives.size <= CHOICE_ERROR_DETAIL_LIMIT
+                            "Expected one of #{alternatives_inspect}"
+                          else
+                            "Expected one of #{@alternatives.size} alternatives"
+                          end
+      end
+
+      # Inspect that survives branches whose own #inspect raises
+      def alternatives_inspect
+        @alternatives.inspect
+      rescue StandardError
+        "[#{@alternatives.map { |atom| atom_inspect(atom) }.join(', ')}]"
+      end
+
+      def atom_inspect(atom)
+        atom.inspect
+      rescue StandardError
+        atom.class.name || atom.class.to_s
+      end
+
+      # Returns [prefix, exact]. prefix is a literal string every successful
+      # match of the atom is guaranteed to start with (nil when none can be
+      # proven). exact is true only when the atom matches exactly that literal
+      # and nothing else, so a parent sequence may keep appending literals
+      # from subsequent parts. A partial prefix (exact: false) is still a
+      # sound index key on its own, but nothing may be appended after it.
+      def static_literal_prefix(atom, seen = {})
+        object_id = atom.object_id
+        return [nil, false] if seen[object_id]
+
+        seen[object_id] = true
+        marked = true
+
+        if atom.instance_of?(Parsanol::Atoms::Str)
+          [atom.str, true]
+        elsif atom.instance_of?(Parsanol::Atoms::Named)
+          static_literal_prefix(atom.parslet, seen)
+        elsif atom.instance_of?(Parsanol::Atoms::Entity)
+          parslet = static_entity_parslet(atom)
+          parslet ? static_literal_prefix(parslet, seen) : [nil, false]
+        elsif atom.instance_of?(Parsanol::Atoms::Sequence)
+          static_sequence_literal_prefix(atom, seen)
+        else
+          [nil, false]
+        end
+      ensure
+        # Only clear markers set by this frame; an early return for an already
+        # seen atom must not remove an ancestor's recursion guard.
+        seen.delete(object_id) if marked
+      end
+
+      def static_sequence_literal_prefix(atom, seen)
+        prefix = +""
+        exact = true
+
+        atom.parslets.each do |part|
+          part_prefix, part_exact = static_literal_prefix(part, seen)
+
+          if part_prefix.nil?
+            exact = false
+            break
+          end
+
+          prefix << part_prefix
+
+          # A partial part may match more input after its own prefix, so
+          # literals from later parts are not guaranteed to follow at this
+          # offset — appending them would over-claim and mis-prune branches.
+          unless part_exact
+            exact = false
+            break
+          end
+        end
+
+        return [nil, false] if prefix.empty?
+
+        [prefix, exact]
+      end
+
+      # Resolves an Entity's rule block without letting a misbehaving block
+      # break index construction (unresolvable entities stay unindexed)
+      def static_entity_parslet(atom)
+        atom.parslet
+      rescue StandardError, NotImplementedError
+        nil
       end
     end
   end

data/lib/parsanol/atoms/capture.rb

@@ -22,7 +22,10 @@ module Parsanol
         success, result = @inner_atom.apply(source, context, consume_all)
 
         if success
-          # Flatten and store the captured value in context
+          # Flatten and store the captured value in context. The write mutates
+          # parse state, so enclosing composite results must not be memoized —
+          # a replay would skip re-storing the capture.
+          context.mark_cache_unsafe!
           flattened = flatten(result)
           context.captures[@capture_key] = flattened
         end

data/lib/parsanol/atoms/context.rb

@@ -18,14 +18,21 @@ module Parsanol
     # Inspired by packrat parsing memoization and incremental parsing techniques.
     #
     class Context
-      # Per-parser cache size thresholds based on profiling different grammar types
-      # Different grammars benefit from caching at different input sizes
+      # Threshold for parser classes without a profiled entry below: recursive
+      # parser-class grammars can need memoization even for tiny inputs.
+      PARSER_DEFAULT_THRESHOLD = 0
+
+      # Threshold for plain atom-level contexts (no parser class): memoization
+      # only pays off past this input size, so small parses skip the overhead.
+      DEFAULT_THRESHOLD = 1000
+
+      # Per-parser cache size thresholds based on profiling different grammar types.
       PARSER_CACHE_LIMITS = {
         "JsonParser" => 10_000,      # JSON needs large inputs to benefit
+        "JsonParsanolParser" => 10_000,
         "ErbParser" => 800,          # ERB benefits earlier
         "CalcParser" => 2000,        # Calculator has low repetition
         "SentenceParser" => 5000,    # Linear grammar, minimal benefit
-        :default => 1000,
       }.freeze
 
       # Number of observed backtrack events before packrat caching engages.
@@ -73,6 +80,12 @@ module Parsanol
           require "parsanol/edit_tracker"
           @interval_trees = Hash.new { |h, k| h[k] = Parsanol::IntervalTree.new }
           @edits = Parsanol::EditTracker.new
+          # Composite [:tree_memo, key] keys keep repetition tree-memo entries
+          # in a namespace separate from try_with_interval entries (raw
+          # +/-object_id integers) within the same @interval_trees hash.
+          # Memoized because tree-memo lookups sit on hot paths and would
+          # otherwise allocate a fresh array per query/store.
+          @tree_memo_keys = Hash.new { |h, k| h[k] = [:tree_memo, k].freeze }
         end
 
         # Cut operator support for aggressive eviction
@@ -82,14 +95,35 @@ module Parsanol
         threshold = adaptive_cache_threshold
         if threshold.nil? && parser_class
           name = parser_class.name&.split("::")&.last
-          threshold = PARSER_CACHE_LIMITS[name] || PARSER_CACHE_LIMITS[:default]
+          threshold = PARSER_CACHE_LIMITS.fetch(name, PARSER_DEFAULT_THRESHOLD)
         end
-        threshold ||= PARSER_CACHE_LIMITS[:default]
+        threshold = DEFAULT_THRESHOLD if threshold.nil?
 
         @adaptive_threshold = threshold
         @input_len = nil
         @caching_active = false
         @backtrack_events = 0
+
+        # Monotonic counter of cache-unsafe events (dynamic-atom evaluations
+        # and capture writes). A result whose computation bumped this counter
+        # depends on mutable parse state, so replaying it from any cache could
+        # change semantics — such results are never stored. This keeps cache
+        # thresholds performance-only: dynamic{} blocks are always
+        # re-evaluated, as their documentation promises.
+        @cache_unsafe_events = 0
+      end
+
+      # @return [Integer] monotonic count of cache-unsafe events so far
+      attr_reader :cache_unsafe_events
+
+      # Records that mutable parse state was read or written (dynamic atom
+      # evaluated, capture stored). Called by atoms; results computed across
+      # such events are excluded from memoization.
+      #
+      # @return [Integer] the updated event count
+      #
+      def mark_cache_unsafe!
+        @cache_unsafe_events += 1
       end
 
       # Attempts to parse using memoization. Returns cached result if available,
@@ -114,8 +148,7 @@ module Parsanol
         return try_uncached_probe(atom, src, must_consume_all) unless @caching_active
 
         pos = src.bytepos
-        key = atom.object_id
-        entry = @memo[pos]
+        key = scoped_cache_key(atom, must_consume_all)
 
         # Periodic cache eviction to prevent unbounded growth
         if pos > @furthest_pos
@@ -129,25 +162,30 @@ module Parsanol
           end
         end
 
-        # Check for cache hit (avoid default-block Hash allocation per probe)
-        if entry&.key?(key)
-          @hit_stats[key] += 1
-          outcome, delta = entry[key]
+        # Check for cache hit (scoped, with the Parslet-compatible
+        # prefix-success fallback for ordered choice)
+        cached_key = cached_entry_key(@memo[pos], atom, must_consume_all)
+        if cached_key
+          @hit_stats[cached_key] += 1
+          outcome, delta = @memo[pos][cached_key]
           src.bytepos = pos + delta
           return outcome
         end
 
         # Cache miss - execute and store
         @miss_stats[key] += 1
+        unsafe_before = @cache_unsafe_events
         outcome = atom.try(src, self, must_consume_all)
         delta = src.bytepos - pos
 
-        # Only cache if beneficial (heuristic)
-        attempts = @hit_stats[key] + @miss_stats[key]
-        if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
-          (@memo[pos] ||= {})[key] =
-            [outcome,
-             delta]
+        # Never store results spanning cache-unsafe events (dynamic
+        # blocks, capture writes); keep the beneficial-attempts
+        # heuristic to avoid one-shot memo pollution.
+        if storable_outcome?(key, outcome, unsafe_before)
+          attempts = @hit_stats[key] + @miss_stats[key]
+          if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
+            (@memo[pos] ||= {})[key] = [outcome, delta]
+          end
         end
 
         outcome
@@ -162,27 +200,25 @@ module Parsanol
       #
       def try_with_interval(atom, src, must_consume_all)
         pos = src.bytepos
-        key = atom.object_id
-
-        tree = @interval_trees[key]
-        cached = tree.query_exact(pos, pos)
+        key = scoped_cache_key(atom, must_consume_all)
+        cached_key, cached = cached_interval_entry(atom, pos,
+                                                   must_consume_all)
 
         if cached
-          @hit_stats[key] += 1
+          @hit_stats[cached_key] += 1
           outcome, delta = cached
           src.bytepos = pos + delta
           return outcome
         end
 
         @miss_stats[key] += 1
+        unsafe_before = @cache_unsafe_events
         outcome = atom.try(src, self, must_consume_all)
         delta = src.bytepos - pos
         end_pos = pos + delta
 
-        attempts = @hit_stats[key] + @miss_stats[key]
-        if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
-          tree.insert(pos, end_pos,
-                      [outcome, delta])
+        if storable_outcome?(key, outcome, unsafe_before)
+          @interval_trees[key].insert(pos, end_pos, [outcome, delta])
         end
 
         outcome
@@ -212,6 +248,14 @@ module Parsanol
         ERROR_RESULT
       end
 
+      # Checks if this context is collecting diagnostic errors.
+      #
+      # @return [Boolean] true when an error reporter is attached
+      #
+      def reporting?
+        !@reporter.nil?
+      end
+
       # Reports a successful parse.
       #
       # @return [Array(Boolean, Object)] success result tuple
@@ -297,10 +341,8 @@ module Parsanol
       def query_tree_memo(key, start_pos)
         return nil unless @use_intervals
 
-        tree = @interval_trees[key]
-        matches = tree.query_overlapping(start_pos, start_pos + 1)
-        found = matches.find { |interval, _| interval[0] == start_pos }
-        found ? found[1] : nil
+        tree = @interval_trees[tree_memo_cache_key(key)]
+        tree.query_starting_at(start_pos).first
       end
 
       # Stores a result in the interval cache.
@@ -313,7 +355,23 @@ module Parsanol
       def store_tree_memo(key, start_pos, values, end_pos)
         return unless @use_intervals
 
-        @interval_trees[key].insert(start_pos, end_pos, [values, end_pos])
+        @interval_trees[tree_memo_cache_key(key)].insert(start_pos, end_pos,
+                                                         [values, end_pos])
+      end
+
+      # Removes tree-memo entries overlapping [start_pos, end_pos) for a key.
+      # Used when a cached entry is detected stale for the current input so a
+      # freshly stored result is not shadowed by the old one.
+      #
+      # @param key [Integer] cache key
+      # @param start_pos [Integer] start position
+      # @param end_pos [Integer] end position
+      #
+      def evict_tree_memo(key, start_pos, end_pos)
+        return unless @use_intervals
+
+        @interval_trees[tree_memo_cache_key(key)].delete_overlapping(start_pos,
+                                                                     end_pos)
       end
 
       # Marks a cut position for aggressive cache eviction.
@@ -343,6 +401,119 @@ module Parsanol
         outcome
       end
 
+      def cached_entry_key(cache, atom, must_consume_all)
+        key = scoped_cache_key(atom, must_consume_all)
+        return key if cache.key?(key)
+
+        return nil unless prefix_success_fallback?(atom, must_consume_all)
+
+        shared_key = shared_cache_key(atom)
+        return shared_key if successful_prefix_entry?(cache[shared_key])
+
+        nil
+      end
+
+      def cached_interval_entry(atom, pos, must_consume_all)
+        key = scoped_cache_key(atom, must_consume_all)
+        cached = @interval_trees[key].query_exact(pos, pos)
+        return [key, cached] if cached
+
+        cached = cached_interval_success_starting_at(key, pos)
+        return [key, cached] if cached
+
+        if prefix_success_fallback?(atom, must_consume_all)
+          shared_key = shared_cache_key(atom)
+          shared = cached_interval_success_starting_at(shared_key, pos)
+          return [shared_key, shared] if shared
+        end
+
+        [nil, nil]
+      end
+
+      def cached_interval_success_starting_at(key, pos)
+        @interval_trees[key].query_starting_at(pos).find do |entry|
+          successful_prefix_entry?(entry)
+        end
+      end
+
+      def tree_memo_cache_key(key)
+        @tree_memo_keys[key]
+      end
+
+      def try_with_prefix_success_cache(atom, src, must_consume_all)
+        pos = src.bytepos
+        shared_key = shared_cache_key(atom)
+
+        if prefix_success_fallback?(atom, must_consume_all)
+          entry = @memo[pos][shared_key]
+          if successful_prefix_entry?(entry)
+            outcome, delta = entry
+            src.bytepos = pos + delta
+            return outcome
+          end
+        end
+
+        unsafe_before = @cache_unsafe_events
+        outcome = atom.try(src, self, must_consume_all)
+
+        if @cache_unsafe_events == unsafe_before &&
+            !must_consume_all && outcome.first &&
+            share_prefix_success_cache?(atom)
+          delta = src.bytepos - pos
+          # This path intentionally keeps only shared prefix successes while
+          # full memoization is inactive. That preserves Parslet ordered-choice
+          # semantics without turning small parses into fully memoized parses.
+          @memo[pos][shared_key] = [outcome, delta]
+        end
+
+        outcome
+      end
+
+      def scoped_cache_key(atom, must_consume_all)
+        must_consume_all ? strict_cache_key(atom) : shared_cache_key(atom)
+      end
+
+      def strict_cache_key(atom)
+        -atom.object_id
+      end
+
+      def shared_cache_key(atom)
+        atom.object_id
+      end
+
+      # True when a non-strict attempt may fall back to a shared
+      # prefix-success entry for this atom. A strict (consume-all)
+      # attempt must never replay a shared entry: parslet semantics
+      # re-execute the ordered choice, and only a fresh parse can take
+      # a longer alternative when the prefix replay starves the
+      # consume-all recheck (parsanol-ruby#22).
+      def prefix_success_fallback?(atom, must_consume_all)
+        !must_consume_all && share_prefix_success_cache?(atom)
+      end
+
+      # An outcome may be memoized only when no cache-unsafe event (dynamic
+      # evaluation, capture write) happened while computing it, and the
+      # attempts heuristic deems the entry worthwhile.
+      def storable_outcome?(key, outcome, unsafe_before)
+        return false unless @cache_unsafe_events == unsafe_before
+
+        outcome.first ||
+          @hit_stats[key] + @miss_stats[key] <= @min_hits_for_cache ||
+          @hit_stats[key].positive?
+      end
+
+      def successful_prefix_entry?(entry)
+        entry && entry[0].is_a?(Array) && entry[0].first == true
+      end
+
+      # Entity, Named, and Ignored delegate to wrapped atoms before this cache
+      # lookup, so only cache-participating built-ins belong in this whitelist.
+      def share_prefix_success_cache?(atom)
+        atom.instance_of?(Parsanol::Atoms::Alternative) ||
+          atom.instance_of?(Parsanol::Atoms::Repetition) ||
+          atom.instance_of?(Parsanol::Atoms::Sequence)
+      end
+
       # Lookup cached result (uses object_id for speed)
       def lookup(atom, pos)
         @memo[pos][atom.object_id]

data/lib/parsanol/atoms/dynamic.rb

@@ -23,6 +23,11 @@ module Parsanol
       end
 
       def try(source, context, consume_all)
+        # The block reads mutable parse state (captures), so no enclosing
+        # composite result may be memoized — replaying it would skip this
+        # re-evaluation.
+        context.mark_cache_unsafe!
+
         # Phase 55: Cache @block ivar to reduce lookup overhead
         block = @block
         result = block.call(source, context)

data/lib/parsanol/atoms/repetition.rb

@@ -180,13 +180,15 @@ module Parsanol
         # Pre-size for the tag + a modest run; Array growth is amortized.
         result = Array.new([@max || 8, 8].min + 1)
         result[0] = @result_tag
-        last_error = nil
+        last_failure = nil
 
         loop do
           success, value = @parslet.apply(source, context, false)
-          last_error = value
 
-          break unless success
+          unless success
+            last_failure = value
+            break
+          end
 
           occurrence += 1
           result[occurrence] = value
@@ -198,12 +200,13 @@ module Parsanol
         if occurrence < @min
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
-                                [last_error])
+                                failure_children(last_failure))
         end
 
         # Check complete consumption
         if consume_all && source.chars_left.positive?
-          return context.err(self, source, @extra_error, [last_error])
+          return context.err(self, source, @extra_error,
+                             failure_children(last_failure))
         end
 
         # Trim to the actual filled length (tag + occurrence matches).
@@ -217,11 +220,27 @@ module Parsanol
         cache_key = object_id
 
         # Check cache
+        replayed_strict_miss = false
         cached = context.query_tree_memo(cache_key, start_pos)
         if cached
           values, end_pos = cached
           source.bytepos = end_pos
-          return ok([@result_tag] + values)
+          if source.bytepos == end_pos
+            unless consume_all && source.chars_left.positive?
+              return ok([@result_tag] + values)
+            end
+
+            # The cached prefix cannot satisfy consume-all. Re-parse instead
+            # of erroring from the replay so diagnostics match the uncached
+            # path; remember not to re-store the entry we already have.
+            replayed_strict_miss = true
+          else
+            # Entry points past the end of the current input — stale (context
+            # reused across inputs). Evict it so the re-parsed result can be
+            # stored and found by later parses instead of being shadowed.
+            context.evict_tree_memo(cache_key, start_pos, end_pos)
+          end
+          source.bytepos = start_pos
         end
 
         # Parse and cache
@@ -231,13 +250,16 @@ module Parsanol
 
         positions = context.acquire_array
         positions << start_pos
-        last_error = nil
+        unsafe_before = context.cache_unsafe_events
+        last_failure = nil
 
         loop do
           success, value = @parslet.apply(source, context, false)
-          last_error = value
 
-          break unless success
+          unless success
+            last_failure = value
+            break
+          end
 
           occurrence += 1
           result[occurrence] = value
@@ -246,8 +268,21 @@ module Parsanol
           break if @max && occurrence >= @max
         end
 
-        # Cache successful prefix
-        if occurrence.positive?
+        # Check minimum
+        if occurrence < @min
+          context.release_array(positions)
+          source.bytepos = start_pos
+          return context.err_at(self, source, @min_error, start_pos,
+                                failure_children(last_failure))
+        end
+
+        # Cache only after the repetition itself has succeeded. A partial prefix
+        # below the minimum bound is not a valid repetition result to replay,
+        # results computed across cache-unsafe events (dynamic/capture) must be
+        # re-evaluated each time, and a strict-miss replay already has its
+        # entry stored.
+        if occurrence.positive? && !replayed_strict_miss &&
+            context.cache_unsafe_events == unsafe_before
           end_pos = positions[occurrence]
           context.store_tree_memo(cache_key, start_pos,
                                   result[1, occurrence], end_pos)
@@ -258,17 +293,25 @@ module Parsanol
         if occurrence < @min
           source.bytepos = start_pos
           return context.err_at(self, source, @min_error, start_pos,
-                                [last_error])
+                                failure_children(last_failure))
         end
 
         # Check consumption
         if consume_all && source.chars_left.positive?
-          return context.err(self, source, @extra_error, [last_error])
+          return context.err(self, source, @extra_error,
+                             failure_children(last_failure))
         end
 
         result.pop(result.size - occurrence - 1) if result.size > occurrence + 1
         ok(result)
       end
+
+      # A max-bounded loop can end with no inner failure; only attach a child
+      # cause when one exists (a success Slice is not a Cause and would break
+      # ascii_tree rendering).
+      def failure_children(last_failure)
+        last_failure && [last_failure]
+      end
     end
   end
 end

data/lib/parsanol/buffer.rb

@@ -95,7 +95,7 @@ module Parsanol
     #
     def clear!
       # Clear references for GC (keep capacity)
-      @size.upto(@capacity - 1) { |i| @storage[i] = nil }
+      @storage.fill(nil, 0, @size)
       @size = 0
       self
     end

data/lib/parsanol/interval_tree.rb

@@ -9,6 +9,7 @@
 # Performance characteristics:
 # - Insert: O(log n)
 # - Query: O(log n + k) where k is number of overlapping intervals
+# - Query by start position: O(log n + k)
 # - Delete overlapping: O(log n + k)
 #
 module Parsanol
@@ -72,6 +73,27 @@ module Parsanol
       find_exact(@root, low, high)
     end
 
+    # Query for intervals that start at a specific position
+    # @param low [Integer] Start position to match
+    # @return [Array<Object>] Data for intervals whose start equals low
+    def query_starting_at(low)
+      results = []
+      node = @root
+
+      while node
+        if low < node.low
+          node = node.left
+        elsif low > node.low
+          node = node.right
+        else
+          results << node.data
+          node = node.right
+        end
+      end
+
+      results
+    end
+
     # Delete all intervals that overlap with [low, high)
     # Returns array of deleted data
     # @param low [Integer] Start position (inclusive)
@@ -140,16 +162,18 @@ module Parsanol
 
     # Find exact interval match
     def find_exact(node, low, high)
-      return nil if node.nil?
-
-      return node.data if node.low == low && node.high == high
-
-      # Search in appropriate subtree
-      if low < node.low
-        find_exact(node.left, low, high)
-      else
-        find_exact(node.right, low, high)
+      while node
+        return node.data if node.low == low && node.high == high
+
+        # Search in appropriate subtree
+        node = if low < node.low
+                 node.left
+               else
+                 node.right
+               end
       end
+
+      nil
     end
 
     # Delete overlapping intervals recursively

data/lib/parsanol/slice.rb

@@ -53,7 +53,7 @@ module Parsanol
     end
 
     def hash
-      [content, offset].hash
+      [Parsanol::Slice, content].hash
     end
 
     # Delegated methods

data/lib/parsanol/source/line_cache.rb

@@ -10,11 +10,20 @@ module Parsanol
     # Caches line ending positions for quick line/column resolution.
     # Uses binary search for efficient position lookup.
     class LineCache
-      def initialize
+      # Creates a line cache, optionally bound to a one-shot buffer that is
+      # scanned lazily on the first line_and_column call.
+      #
+      # @param buffer [String, nil] input to scan lazily; nil for callers that
+      #   feed windows incrementally via scan_for_line_endings
+      # @param start_offset [Integer] byte offset of the buffer's first byte
+      def initialize(buffer = nil, start_offset = 0)
         # Array of byte offsets where each line ends
         @breaks = []
         @breaks.extend(IntervalLookup)
         @max_scanned = nil
+        @buffer = buffer
+        @start_offset = start_offset
+        @fully_scanned = false
       end
 
       # Converts a byte offset to [line_number, column_number].
@@ -23,6 +32,8 @@ module Parsanol
       # @param position [Integer, #bytepos] the byte offset to convert
       # @return [Array<Integer, Integer>] [line, column] tuple
       def line_and_column(position)
+        scan_buffer_once
+
         position = position.bytepos if position.respond_to?(:bytepos)
 
         line_idx = @breaks.lower_bound_index(position)
@@ -39,7 +50,10 @@ module Parsanol
       end
 
       # Scans a string buffer for line endings and caches their positions.
-      # Avoids re-scanning already processed regions.
+      # Avoids re-scanning already processed regions. Incremental callers must
+      # feed windows of one consistent input in monotonically advancing,
+      # contiguous-or-overlapping order; non-contiguous or out-of-order scans
+      # are skipped where already covered and can miss line endings in gaps.
       #
       # @param start_offset [Integer] the byte offset where buffer starts
       # @param buffer [String] the string content to scan
@@ -47,16 +61,34 @@ module Parsanol
         return unless buffer
 
         scanner = StringScanner.new(buffer)
-        return unless scanner.exist?(/\n/)
-
-        # Skip already-scanned content
-        scanner.pos = @max_scanned - start_offset if @max_scanned && start_offset < @max_scanned
+        if scanner.exist?(/\n/)
+          # Skip already-scanned content. @max_scanned can extend past this
+          # buffer's window (it advances to the end of every scanned buffer),
+          # so clamp to the window to keep the scanner position valid.
+          if @max_scanned && start_offset < @max_scanned
+            scanner.pos = [@max_scanned - start_offset, buffer.bytesize].min
+          end
 
-        # Record all newline positions
-        while scanner.skip_until(/\n/)
-          @max_scanned = start_offset + scanner.pos
-          @breaks << @max_scanned
+          # Record all newline positions
+          while scanner.skip_until(/\n/)
+            @max_scanned = start_offset + scanner.pos
+            @breaks << @max_scanned
+          end
         end
+
+        @max_scanned = [@max_scanned || start_offset, start_offset + buffer.bytesize].max
+      end
+
+      private
+
+      # Scans the one-shot constructor buffer on first use, then releases it
+      # so retained Slices do not pin the entire input string.
+      def scan_buffer_once
+        return if @fully_scanned || !@buffer
+
+        scan_for_line_endings(@start_offset, @buffer)
+        @fully_scanned = true
+        @buffer = nil
       end
     end
 

data/lib/parsanol/source.rb

@@ -38,18 +38,18 @@ module Parsanol
               "Source requires a string-like object (responds to to_str)"
       end
 
-      # Core scanner for input traversal
-      @scanner = StringScanner.new(input)
       @raw_string = input.to_str
 
+      # Core scanner for input traversal
+      @scanner = StringScanner.new(@raw_string)
+
       # Regex cache: maps count n to /(.|$){n}/m pattern
       @regex_cache = Hash.new do |h, count|
         h[count] = Regexp.new("(.|$){#{count}}", Regexp::MULTILINE)
       end
 
       # Line ending cache for position-to-line/column mapping
-      @line_data = LineCache.new
-      @line_data.scan_for_line_endings(0, input)
+      @line_data = LineCache.new(@raw_string)
 
       # Object pools for memory efficiency
       # SlicePool: reduces Slice allocations during matching
@@ -160,6 +160,23 @@ module Parsanol
       @scanner.rest_size
     end
 
+    # Returns the unconsumed input from the current position without advancing.
+    #
+    # @return [String] remaining input
+    #
+    def remaining
+      @scanner.rest
+    end
+
+    # Returns up to byte_count bytes from the current position without advancing.
+    #
+    # @param byte_count [Integer] maximum bytes to read
+    # @return [String] bounded input preview
+    #
+    def peek(byte_count)
+      @scanner.peek(byte_count)
+    end
+
     # Counts characters from current position until a target string.
     # Returns chars_left if target is not found.
     #

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.47"
+  VERSION = "1.3.48"
 end

data/lib/parsanol/vm.rb

@@ -103,7 +103,17 @@ module Parsanol
         return nil if cached == :fallback
         return cached if key?(cache, key)
 
-        cache[key] = compile(atom)
+        # A grammar the compiler cannot compile — e.g. one containing a
+        # lazily resolved Entity that a select-first literal index would
+        # never select — falls back to the interpreter, the source of
+        # truth. The interpreter resolves such entities lazily, so an
+        # unselected branch simply never raises.
+        begin
+          cache[key] = compile(atom)
+        rescue NotImplementedError
+          cache[key] = :fallback
+          nil
+        end
       end
 
       # Marks a grammar as VM-incompatible after a runtime failure.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.47
+  version: 1.3.48
 platform: ruby
 authors:
 - Ribose Inc.