parsanol 3.0.0

data/lib/parsanol/atoms/context.rb

@@ -0,0 +1,351 @@
+module Parsanol::Atoms
+  # Helper class that implements a transient cache that maps position and
+  # parslet object to results. This is used for memoization in the packrat
+  # style.
+  #
+  # Also, error reporter is stored here and error reporting happens through
+  # this class. This makes the reporting pluggable.
+  #
+  class Context
+    # Parser-specific cache thresholds (Session 13)
+    # Based on profiling: different parsers have different cache benefit points
+    # - JSON: High recursion on large files, but medium files (5KB) see overhead
+    # - ERB: Moderate repetition, benefits from cache earlier
+    # - Calc: Lower repetition, needs larger input
+    # - Sentence: Simple linear grammar, minimal cache benefit
+    PARSER_CACHE_THRESHOLDS = {
+      'JsonParser' => 10_000,      # High threshold - json/medium regressed at 1000
+      'ErbParser' => 800,           # Moderate - working well
+      'CalcParser' => 2000,         # Low repetition
+      'SentenceParser' => 5000,     # Linear grammar
+      :default => 1000
+    }.freeze
+    
+    # @param reporter [#err, #err_at] Error reporter (leave empty for default
+    #   reporter)
+    # @param interval_cache [Boolean] Use GPeg-style interval tree caching
+    # @param adaptive_cache_threshold [Integer] Disable caching for inputs smaller than this (bytes)
+    # @param parser_class [Class] Parser class for per-parser threshold selection
+    def initialize(reporter=Parsanol::ErrorReporter::Tree.new, interval_cache: false, adaptive_cache_threshold: nil, parser_class: nil)
+      @cache = Hash.new { |h, k| h[k] = {} }
+      @reporter = reporter
+      @captures = Parsanol::Scope.new
+      @max_position = 0  # Track furthest position for cache eviction
+      @eviction_threshold = 200  # Evict positions more than 200 bytes behind
+      @eviction_counter = 0  # Counter for periodic eviction
+      @eviction_frequency = 100  # Only evict every N position advances
+
+      # Phase 1.3: ArrayPool for reducing GC pressure from array allocations
+      # Arrays are the highest allocation source (74% of memory allocations)
+      # Initialize pool with reasonable size for typical parsing workloads
+      @array_pool = Parsanol::Pools::ArrayPool.new(size: 10000)
+
+      # Phase 2.1: BufferPool for fixed-size buffer pre-allocation
+      # Reduces array allocations through buffer reuse by size class
+      @buffer_pool = Parsanol::Pools::BufferPool.new(pool_size: 100)
+
+      # Selective memoization: track hit/miss rates to only cache beneficial parslets
+      @hit_counts = Hash.new(0)
+      @miss_counts = Hash.new(0)
+      @cache_threshold = 2  # Only cache if we've had 2+ hits
+
+      # GPeg-style interval tree caching (optional)
+      @use_interval_cache = interval_cache
+      if @use_interval_cache
+        require 'parsanol/interval_tree'
+        require 'parsanol/edit_tracker'
+        # Map parslet object_id to interval tree
+        @interval_cache = Hash.new { |h, k| h[k] = Parsanol::IntervalTree.new }
+        # Track edits for lazy position shifts
+        @edit_tracker = Parsanol::EditTracker.new
+      end
+
+      # Cut operator support (Phase 46b)
+      # Track the last cut position to enable aggressive cache eviction
+      @last_cut_position = 0
+
+      # Adaptive caching (Session 12-13): Disable cache for small inputs
+      # Session 13: Per-parser thresholds based on profiling
+      # - JSON medium (5KB) regressed with 1000-byte threshold → raised to 10KB
+      # - Different parsers have different cache benefit points
+      
+      # Determine threshold: explicit > parser-specific > default
+      threshold = adaptive_cache_threshold
+      if threshold.nil? && parser_class
+        # Extract simple class name (e.g., "MyJson::Parser" -> "Parser")
+        parser_name = parser_class.name&.split('::')&.last
+        threshold = PARSER_CACHE_THRESHOLDS[parser_name] || PARSER_CACHE_THRESHOLDS[:default]
+      end
+      threshold ||= PARSER_CACHE_THRESHOLDS[:default]
+      
+      @adaptive_cache_threshold = threshold
+      @input_size = nil  # Will be set on first parse attempt
+      @caching_enabled = nil  # Will be determined based on input size
+    end
+
+    # Caches a parse answer for obj at source.pos. Applying the same parslet
+    # at one position of input always yields the same result, unless the input
+    # has changed.
+    #
+    # We need the entire source here so we can ask for how many characters
+    # were consumed by a successful parse. Imitation of such a parse must
+    # advance the input pos by the same amount of bytes.
+    #
+    def try_with_cache(obj, source, consume_all)
+      # Skip caching entirely for atoms that don't benefit from it
+      unless obj.cached?
+        return obj.try(source, self, consume_all)
+      end
+
+      # Session 12: Adaptive caching based on input size
+      # Determine if caching should be enabled (only on first call)
+      if @caching_enabled.nil?
+        # Get total input size from source
+        input_size = source.bytepos + source.chars_left
+        @input_size = input_size
+        @caching_enabled = input_size >= @adaptive_cache_threshold
+      end
+
+      # For small inputs, skip caching entirely - the overhead exceeds benefit
+      # Profiling shows cache overhead is 15-20% for inputs < 1000 bytes
+      unless @caching_enabled
+        return obj.try(source, self, consume_all)
+      end
+
+      # Phase 55: Cache ivars to reduce lookup overhead in hot method
+      use_interval_cache = @use_interval_cache
+      cache = @cache
+      hit_counts = @hit_counts
+      miss_counts = @miss_counts
+      cache_threshold = @cache_threshold
+
+      # Use interval-based caching if enabled (GPeg-style)
+      if use_interval_cache
+        return try_with_interval_cache(obj, source, consume_all)
+      end
+
+      beg = source.bytepos
+      cache_key = obj.object_id
+
+      # Track furthest position and evict old cache entries PERIODICALLY
+      # In left-to-right parsing, positions far behind won't be revisited
+      if beg > @max_position
+        @max_position = beg
+        eviction_counter = @eviction_counter + 1
+        @eviction_counter = eviction_counter
+
+        # Evict positions that are too far behind current position
+        # This prevents unbounded cache growth (O(n*m) memory issue in packrat)
+        # Phase 42: Only evict periodically instead of on every position advance
+        # This reduces delete_if calls from ~900K to ~9K (100x reduction)
+        if eviction_counter >= @eviction_frequency
+          @eviction_counter = 0
+          min_keep_pos = beg - @eviction_threshold
+          cache.delete_if { |pos, _| pos < min_keep_pos }
+        end
+      end
+
+      # Check if this parslet/position combo is already cached
+      if cache[beg].key?(cache_key)
+        # Cache hit - track it
+        hit_counts[cache_key] += 1
+        result, advance = cache[beg][cache_key]
+        source.bytepos = beg + advance
+        return result
+      end
+
+      # Cache miss - execute the parslet
+      miss_counts[cache_key] += 1
+      result = obj.try(source, self, consume_all)
+      advance = source.bytepos - beg
+
+      # Only cache if this parslet has shown it benefits from caching
+      # (has had multiple hits, or we're still learning about it)
+      total_attempts = hit_counts[cache_key] + miss_counts[cache_key]
+      if total_attempts <= cache_threshold || hit_counts[cache_key] > 0
+        cache[beg][cache_key] = [result, advance]
+      end
+
+      return result
+    end
+
+    # GPeg-style interval-based caching
+    # Caches results keyed by intervals [start, end) rather than single positions
+    # This enables efficient invalidation of changed regions during incremental parsing
+    def try_with_interval_cache(obj, source, consume_all)
+      beg = source.bytepos
+      cache_key = obj.object_id
+
+      # Try to find exact match in interval tree
+      tree = @interval_cache[cache_key]
+      result_data = tree.query_exact(beg, beg)  # Start with point query
+
+      if result_data
+        # Exact match found - restore result
+        @hit_counts[cache_key] += 1
+        result, advance = result_data
+        source.bytepos = beg + advance
+        return result
+      end
+
+      # No exact match - execute the parslet
+      @miss_counts[cache_key] += 1
+      result = obj.try(source, self, consume_all)
+      advance = source.bytepos - beg
+      end_pos = beg + advance
+
+      # Store in interval tree: [start, end) -> [result, advance]
+      # Only cache if beneficial (selective memoization)
+      total_attempts = @hit_counts[cache_key] + @miss_counts[cache_key]
+      if total_attempts <= @cache_threshold || @hit_counts[cache_key] > 0
+        tree.insert(beg, end_pos, [result, advance])
+      end
+
+      return result
+    end
+
+    # Pre-allocated constants to avoid repeated array allocations
+    # These are the most common return values during parsing
+    SUCCESS_NIL = [true, nil].freeze
+    ERROR_NIL = [false, nil].freeze
+
+    # Report an error at a given position.
+    # @see ErrorReporter
+    #
+    def err_at(*args)
+      return [false, @reporter.err_at(*args)] if @reporter
+      ERROR_NIL
+    end
+
+    # Report an error.
+    # @see ErrorReporter
+    #
+    def err(*args)
+      return [false, @reporter.err(*args)] if @reporter
+      ERROR_NIL
+    end
+
+    # Report a successful parse.
+    # @see ErrorReporter::Contextual
+    #
+    def succ(*args)
+      # The default error reporter (Tree) has an empty succ method that returns nil
+      # So for the common case (no reporter or default reporter), use pre-allocated constant
+      return SUCCESS_NIL unless @reporter
+      result = @reporter.succ(*args)
+      return SUCCESS_NIL if result.nil?
+      [true, result]
+    end
+
+    # Returns the current captures made on the input (see
+    # Parsanol::Atoms::Base#capture). Use as follows:
+    #
+    #   context.captures[:foobar] # => returns capture :foobar
+    #
+    attr_reader :captures
+
+    # Phase 1.3: Expose ArrayPool for array acquisition/release
+    # @return [Parsanol::Pools::ArrayPool] The array pool instance
+    attr_reader :array_pool
+
+    # Acquire an array from the pool.
+    # Returns a cleared, empty array ready for use.
+    #
+    # @return [Array] An empty array from the pool
+    def acquire_array
+      @array_pool.acquire
+    end
+
+    # Release an array back to the pool.
+    # The array will be cleared and made available for reuse.
+    #
+    # @param array [Array] The array to return to the pool
+    # @return [Boolean] true if returned to pool, false if discarded
+    def release_array(array)
+      @array_pool.release(array)
+    end
+
+    # Phase 2.1: Expose BufferPool for buffer acquisition/release
+    # @return [Parsanol::Pools::BufferPool] The buffer pool instance
+    attr_reader :buffer_pool
+
+    # Acquire a buffer from the pool with specified minimum capacity.
+    #
+    # @param size [Integer] Minimum required capacity
+    # @return [Parsanol::Buffer] Buffer with capacity >= size
+    def acquire_buffer(size:)
+      @buffer_pool.acquire(size: size)
+    end
+
+    # Release a buffer back to the pool.
+    # The buffer will be cleared and made available for reuse.
+    #
+    # @param buffer [Parsanol::Buffer] The buffer to return to the pool
+    # @return [Boolean] true if returned to pool, false if discarded
+    def release_buffer(buffer)
+      @buffer_pool.release(buffer)
+    end
+
+    # Starts a new scope. Use the #scope method of Parsanol::Atoms::DSL
+    # to call this.
+    #
+    def scope
+      captures.push
+      yield
+    ensure
+      captures.pop
+    end
+
+    # GPeg-style tree memoization support
+    # Check if tree memoization is enabled
+    def use_tree_memoization?
+      @use_interval_cache
+    end
+
+    # Query tree memo cache for a given key and position
+    # Returns [values, end_pos] if found, nil otherwise
+    def query_tree_memo(cache_key, start_pos)
+      return nil unless @use_interval_cache
+      tree = @interval_cache[cache_key]
+      # Query for any intervals that overlap with [start_pos, start_pos+1)
+      # This will find intervals that start at start_pos
+      overlapping = tree.query_overlapping(start_pos, start_pos + 1)
+      # Find exact match where interval starts at start_pos
+      result = overlapping.find { |interval, _data| interval[0] == start_pos }
+      result ? result[1] : nil
+    end
+
+    # Store tree memo: cache array of values for repetition
+    def store_tree_memo(cache_key, start_pos, values, end_pos)
+      return unless @use_interval_cache
+      tree = @interval_cache[cache_key]
+      tree.insert(start_pos, end_pos, [values, end_pos])
+    end
+
+    # Cut operator support (Phase 46b)
+    # Called when a cut operator succeeds. This enables aggressive cache eviction
+    # by marking that we won't backtrack before this position.
+    #
+    # @param position [Integer] The position where the cut occurred
+    def cut!(position)
+      @last_cut_position = position
+
+      # Aggressively evict all cache entries before the cut position
+      # This is safe because we won't backtrack past the cut point
+      # This is the key to achieving O(1) space complexity with cuts
+      @cache.delete_if { |pos, _| pos < position }
+    end
+
+  private
+    # NOTE These methods use #object_id directly, since that seems to bring the
+    # most performance benefit. This is a hot spot; going through
+    # Atoms::Base#hash doesn't yield as much.
+    #
+    def lookup(obj, pos)
+      @cache[pos][obj.object_id]
+    end
+    def set(obj, pos, val)
+      @cache[pos][obj.object_id] = val
+    end
+  end
+end

data/lib/parsanol/atoms/context_optimized.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# Experimental: Position-based cache eviction for Context
+# Based on PEG theory: in linear parsing, positions behind current position
+# will never be revisited, so we can evict them to reduce memory
+
+module Parsanol
+  module Atoms
+    class Context
+      # Add position tracking for cache eviction
+      attr_reader :current_position
+
+      def try_with_cache(obj, source, consume_all)
+        unless obj.cached?
+          return obj.try(source, self, consume_all)
+        end
+
+        key = source.pos
+        @current_position = key
+        atom_cache = @cache[obj]
+
+        # Try to fetch from cache
+        if atom_cache.key?(key)
+          return atom_cache.fetch(key)
+        end
+
+        # Cache miss - compute result
+        result = obj.try(source, self, consume_all)
+        atom_cache[key] = result
+
+        # Evict old positions if cache is getting large
+        # Keep only positions within a window of current position
+        if atom_cache.size > 100
+          min_pos = key - 50 # Keep 50 positions behind
+          atom_cache.delete_if { |pos, _| pos < min_pos }
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/custom.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # Base class for creating custom parser atoms.
+    #
+    # Custom atoms allow extending Parsanol with domain-specific matching logic
+    # that cannot be expressed with the built-in combinators.
+    #
+    # @example Custom atom for matching indentation-sensitive content
+    #   class IndentAtom < Parsanol::Atoms::Custom
+    #     def initialize(expected_indent)
+    #       @expected_indent = expected_indent
+    #       super()
+    #     end
+    #
+    #     # Required: Implement try_match
+    #     def try_match(source, context, consume_all)
+    #       pos = source.pos
+    #       indent = count_indent(source)
+    #
+    #       if indent == @expected_indent
+    #         content = read_until_newline(source)
+    #         [true, content]
+    #       else
+    #         source.pos = pos  # Restore position on failure
+    #         [false, nil]
+    #       end
+    #     end
+    #
+    #     private
+    #
+    #     def count_indent(source)
+    #       # ... implementation ...
+    #     end
+    #   end
+    #
+    #   # Usage in parser
+    #   class MyParser < Parsanol::Parser
+    #     rule(:indented_line) { IndentAtom.new(2) }
+    #   end
+    #
+    class Custom < Base
+      # Required: Implement this method to define matching behavior
+      #
+      # @param source [Parsanol::Source] The input source with position tracking
+      # @param context [Parsanol::Atoms::Context] Parse context for memoization
+      # @param consume_all [Boolean] If true, must consume entire input
+      # @return [Array<Boolean, Object>] Tuple of [success, result]
+      #   - success: true if match succeeded, false otherwise
+      #   - result: matched value on success, nil on failure
+      #
+      # @note You MUST restore source.bytepos on failure for proper backtracking
+      #
+      def try_match(source, context, consume_all)
+        raise NotImplementedError,
+          "Custom atoms must implement #try_match(source, context, consume_all)"
+      end
+
+      # Override of Base#try that delegates to try_match
+      # Handles error reporting and result wrapping
+      #
+      # @api private
+      def try(source, context, consume_all)
+        success, result = try_match(source, context, consume_all)
+
+        if success
+          [true, result]
+        else
+          # Generate error cause for reporting
+          context.err(
+            self,
+            source,
+            "Failed to match custom atom: #{self.class.name}"
+          )
+        end
+      end
+
+      # Optional: Override to provide first set for optimization
+      # Returns the set of characters/strings this atom can match at start
+      #
+      # @return [Set<String>, nil] First set, or nil if not determinable
+      def first_set
+        nil  # Unknown by default
+      end
+
+      # Optional: Override to enable caching for this atom
+      # Return false for context-dependent matching (e.g., indentation)
+      #
+      # @return [Boolean] true if atom can be cached
+      def cacheable?
+        true
+      end
+
+      # Optional: Override to provide custom serialization for native parser
+      # Return nil if atom cannot be serialized (must use pure Ruby mode)
+      #
+      # @return [Hash, nil] JSON-serializable representation
+      def to_native_format
+        nil  # Not serializable by default
+      end
+
+      # Override to_s_inner for debug printing
+      # @api private
+      def to_s_inner(prec = nil)
+        "custom(#{self.class.name})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/cut.rb

@@ -0,0 +1,62 @@
+# Cut operator for PEG grammars
+#
+# A cut operator (↑) instructs the parser to discard backtrack information
+# at a specific point. This enables more aggressive cache eviction and can
+# reduce space complexity from O(n) to O(1).
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+# Example:
+#
+#   rule(:statement) {
+#     str('if').cut >> condition >> then_clause |
+#     str('while').cut >> condition >> body |
+#     str('print').cut >> expression
+#   }
+#
+# After 'if' succeeds, the cut discards backtrack info for 'while' and 'print'.
+# This means if the parse fails later in the 'if' branch, we won't try the
+# other alternatives.
+#
+class Parsanol::Atoms::Cut < Parsanol::Atoms::Base
+  attr_reader :parslet
+
+  def initialize(parslet)
+    super()
+    @parslet = parslet
+  end
+
+  def try(source, context, consume_all)
+    # First, try to match the parslet
+    success, value = parslet.apply(source, context, consume_all)
+
+    return [success, value] unless success
+
+    # On success, signal to context that a cut has occurred
+    # This allows the context to:
+    # 1. Mark the current position as a cut point
+    # 2. Empty the backtrack stack (we won't backtrack past here)
+    # 3. Aggressively evict cache entries before this position
+    if context.respond_to?(:cut!)
+      context.cut!(source.bytepos)
+    end
+
+    return [success, value]
+  end
+
+  # Cut doesn't need caching - it's a thin wrapper
+  def cached?
+    false
+  end
+
+  def to_s_inner(prec)
+    "#{parslet.to_s(prec)}↑"
+  end
+
+  # FIRST set of cut is same as wrapped parslet
+  # Cut doesn't change matching behavior, only affects backtracking
+  def compute_first_set
+    parslet.first_set
+  end
+end

data/lib/parsanol/atoms/dsl.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+
+# A mixin module that defines operations that can be called on any subclass
+# of Parsanol::Atoms::Base. These operations make parslets atoms chainable and
+# allow combination of parslet atoms to form bigger parsers.
+#
+# Example:
+#
+#   str('foo') >> str('bar')
+#   str('f').repeat
+#   any.absent?               # also called The Epsilon
+#
+module Parsanol::Atoms::DSL
+  # Construct a new atom that repeats the current atom min times at least and
+  # at most max times. max can be nil to indicate that no maximum is present.
+  #
+  # Example:
+  #   # match any number of 'a's
+  #   str('a').repeat
+  #
+  #   # match between 1 and 3 'a's
+  #   str('a').repeat(1,3)
+  #
+  def repeat(min=0, max=nil)
+    Parsanol::Atoms::Repetition.new(self, min, max)
+  end
+
+  # Returns a new parslet atom that is only maybe present in the input. This
+  # is synonymous to calling #repeat(0,1). Generated tree value will be
+  # either nil (if atom is not present in the input) or the matched subtree.
+  #
+  # Example:
+  #   str('foo').maybe
+  #
+  def maybe
+    Parsanol::Atoms::Repetition.new(self, 0, 1, :maybe)
+  end
+
+  # Returns a new parslet atom that will not show up in the output. This
+  # is synonymous to calling #repeat(0,1). Generated tree value will always be
+  # nil.
+  #
+  # Example:
+  #   str('foo').ignore
+  #
+  def ignore
+    Parsanol::Atoms::Ignored.new(self)
+  end
+
+  # Chains two parslet atoms together as a sequence.
+  #
+  # Example:
+  #   str('a') >> str('b')
+  #
+  def >>(parslet)
+    Parsanol::Atoms::Sequence.new(self, parslet)
+  end
+
+  # Chains two parslet atoms together to express alternation. A match will
+  # always be attempted with the parslet on the left side first. If it doesn't
+  # match, the right side will be tried.
+  #
+  # Example:
+  #   # matches either 'a' OR 'b'
+  #   str('a') | str('b')
+  #
+  def |(parslet)
+    Parsanol::Atoms::Alternative.new(self, parslet)
+  end
+
+  # Tests for absence of a parslet atom in the input stream without consuming
+  # it.
+  #
+  # Example:
+  #   # Only proceed the parse if 'a' is absent.
+  #   str('a').absent?
+  #
+  def absent?
+    Parsanol::Atoms::Lookahead.new(self, false)
+  end
+
+  # Tests for presence of a parslet atom in the input stream without consuming
+  # it.
+  #
+  # Example:
+  #   # Only proceed the parse if 'a' is present.
+  #   str('a').present?
+  #
+  def present?
+    Parsanol::Atoms::Lookahead.new(self, true)
+  end
+
+  # Marks a parslet atom as important for the tree output. This must be used
+  # to achieve meaningful output from the #parse method.
+  #
+  # Example:
+  #   str('a').as(:b) # will produce {:b => 'a'}
+  #
+  def as(name)
+    Parsanol::Atoms::Named.new(self, name)
+  end
+
+  # Captures a part of the input and stores it under the name given. This
+  # is very useful to create self-referential parses. A capture stores
+  # the result of its parse (may be complex) on a successful parse action.
+  #
+  # Example:
+  #   str('a').capture(:b)  # will store captures[:b] == 'a'
+  #
+  def capture(name)
+    Parsanol::Atoms::Capture.new(self, name)
+  end
+
+  # Marks this parslet atom as a cut point. After this atom succeeds,
+  # the parser will discard backtrack information, enabling O(1) space
+  # complexity. Use with caution: cuts prevent backtracking to alternative
+  # branches.
+  #
+  # Example:
+  #   str('if').cut >> condition >> then_clause |
+  #   str('while') >> condition >> body
+  #
+  # If 'if' matches, we commit to the first branch. If condition or then_clause
+  # fail, we won't try the 'while' alternative.
+  #
+  def cut
+    Parsanol::Atoms::Cut.new(self)
+  end
+end