parsanol 1.0.1-aarch64-linux

data/lib/parsanol/atoms/context.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # Parsing context that coordinates memoization caching, error reporting,
+    # and resource pooling. Created fresh for each parse operation.
+    #
+    # Key responsibilities:
+    # - Packrat-style memoization (caching parse results by position+atom)
+    # - Pluggable error reporting through reporter interface
+    # - Object pooling for arrays and buffers to reduce GC pressure
+    # - Adaptive caching based on input size
+    #
+    # @example Basic usage
+    #   ctx = Context.new(reporter)
+    #   result = ctx.try_with_cache(parser, source, true)
+    #
+    # Inspired by packrat parsing memoization and incremental parsing techniques.
+    #
+    class Context
+      # Per-parser cache size thresholds based on profiling different grammar types
+      # Different grammars benefit from caching at different input sizes
+      PARSER_CACHE_LIMITS = {
+        'JsonParser' => 10_000,      # JSON needs large inputs to benefit
+        'ErbParser' => 800,          # ERB benefits earlier
+        'CalcParser' => 2000,        # Calculator has low repetition
+        'SentenceParser' => 5000,    # Linear grammar, minimal benefit
+        :default => 1000
+      }.freeze
+
+      # Creates a new parsing context.
+      #
+      # @param error_reporter [#err, #err_at] error reporter instance
+      # @param interval_cache: [Boolean] enable GPeg-style interval caching
+      # @param adaptive_cache_threshold: [Integer, nil] minimum input size for caching
+      # @param parser_class: [Class, nil] parser class for threshold selection
+      #
+      def initialize(error_reporter = Parsanol::ErrorReporter::Tree.new,
+                     interval_cache: false,
+                     adaptive_cache_threshold: nil,
+                     parser_class: nil)
+        # Core memoization cache: position -> { atom_id -> [result, advance] }
+        @memo = Hash.new { |h, k| h[k] = {} }
+
+        # Error reporting delegate
+        @reporter = error_reporter
+
+        # Capture scope for variable bindings
+        @captures = Parsanol::Scope.new
+
+        # Cache eviction state
+        @furthest_pos = 0
+        @evict_threshold = 200
+        @evict_counter = 0
+        @evict_interval = 100
+
+        # Object pools for reducing allocations
+        @array_pool = Parsanol::Pools::ArrayPool.new(size: 10_000)
+        @buffer_pool = Parsanol::Pools::BufferPool.new(pool_size: 100)
+
+        # Selective memoization tracking
+        @hit_stats = Hash.new(0)
+        @miss_stats = Hash.new(0)
+        @min_hits_for_cache = 2
+
+        # Optional GPeg-style interval caching
+        @use_intervals = interval_cache
+        if @use_intervals
+          require 'parsanol/interval_tree'
+          require 'parsanol/edit_tracker'
+          @interval_trees = Hash.new { |h, k| h[k] = Parsanol::IntervalTree.new }
+          @edits = Parsanol::EditTracker.new
+        end
+
+        # Cut operator support for aggressive eviction
+        @cut_pos = 0
+
+        # Determine adaptive cache threshold
+        threshold = adaptive_cache_threshold
+        if threshold.nil? && parser_class
+          name = parser_class.name&.split('::')&.last
+          threshold = PARSER_CACHE_LIMITS[name] || PARSER_CACHE_LIMITS[:default]
+        end
+        threshold ||= PARSER_CACHE_LIMITS[:default]
+
+        @adaptive_threshold = threshold
+        @input_len = nil
+        @caching_active = nil
+      end
+
+      # Attempts to parse using memoization. Returns cached result if available,
+      # otherwise executes the parser and caches the result.
+      #
+      # @param atom [Parsanol::Atoms::Base] parser to apply
+      # @param src [Parsanol::Source] input source
+      # @param must_consume_all [Boolean] require complete consumption
+      # @return [Array(Boolean, Object)] parse result tuple
+      #
+      def try_with_cache(atom, src, must_consume_all)
+        # Skip caching for atoms that don't benefit from it
+        return atom.try(src, self, must_consume_all) unless atom.cached?
+
+        # Determine if caching should be active (lazy initialization)
+        if @caching_active.nil?
+          total_len = src.bytepos + src.chars_left
+          @input_len = total_len
+          @caching_active = total_len >= @adaptive_threshold
+        end
+
+        # For small inputs, skip caching overhead
+        return atom.try(src, self, must_consume_all) unless @caching_active
+
+        # Use interval-based caching if enabled
+        return try_with_interval(atom, src, must_consume_all) if @use_intervals
+
+        pos = src.bytepos
+        key = atom.object_id
+
+        # Periodic cache eviction to prevent unbounded growth
+        if pos > @furthest_pos
+          @furthest_pos = pos
+          @evict_counter += 1
+
+          if @evict_counter >= @evict_interval
+            @evict_counter = 0
+            cutoff = pos - @evict_threshold
+            @memo.delete_if { |p, _| p < cutoff }
+          end
+        end
+
+        # Check for cache hit
+        if @memo[pos].key?(key)
+          @hit_stats[key] += 1
+          outcome, delta = @memo[pos][key]
+          src.bytepos = pos + delta
+          return outcome
+        end
+
+        # Cache miss - execute and store
+        @miss_stats[key] += 1
+        outcome = atom.try(src, self, must_consume_all)
+        delta = src.bytepos - pos
+
+        # Only cache if beneficial (heuristic)
+        attempts = @hit_stats[key] + @miss_stats[key]
+        @memo[pos][key] = [outcome, delta] if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
+
+        outcome
+      end
+
+      # GPeg-style interval-based caching for incremental parsing.
+      #
+      # @param atom [Parsanol::Atoms::Base] parser to apply
+      # @param src [Parsanol::Source] input source
+      # @param must_consume_all [Boolean] require complete consumption
+      # @return [Array(Boolean, Object)] parse result tuple
+      #
+      def try_with_interval(atom, src, must_consume_all)
+        pos = src.bytepos
+        key = atom.object_id
+
+        tree = @interval_trees[key]
+        cached = tree.query_exact(pos, pos)
+
+        if cached
+          @hit_stats[key] += 1
+          outcome, delta = cached
+          src.bytepos = pos + delta
+          return outcome
+        end
+
+        @miss_stats[key] += 1
+        outcome = atom.try(src, self, must_consume_all)
+        delta = src.bytepos - pos
+        end_pos = pos + delta
+
+        attempts = @hit_stats[key] + @miss_stats[key]
+        tree.insert(pos, end_pos, [outcome, delta]) if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
+
+        outcome
+      end
+
+      # Pre-allocated result constants
+      SUCCESS_RESULT = [true, nil].freeze
+      ERROR_RESULT = [false, nil].freeze
+
+      # Reports an error at a specific position.
+      #
+      # @return [Array(Boolean, Object)] error result tuple
+      #
+      def err_at(*args)
+        return [false, @reporter.err_at(*args)] if @reporter
+
+        ERROR_RESULT
+      end
+
+      # Reports an error at the current position.
+      #
+      # @return [Array(Boolean, Object)] error result tuple
+      #
+      def err(*args)
+        return [false, @reporter.err(*args)] if @reporter
+
+        ERROR_RESULT
+      end
+
+      # Reports a successful parse.
+      #
+      # @return [Array(Boolean, Object)] success result tuple
+      #
+      def succ(*args)
+        return SUCCESS_RESULT unless @reporter
+
+        val = @reporter.succ(*args)
+        return SUCCESS_RESULT if val.nil?
+
+        [true, val]
+      end
+
+      # @return [Parsanol::Scope] capture variable bindings
+      attr_reader :captures
+
+      # @return [Parsanol::Pools::ArrayPool] array object pool
+      attr_reader :array_pool
+
+      # @return [Parsanol::Pools::BufferPool] buffer object pool
+      attr_reader :buffer_pool
+
+      # Acquires an empty array from the pool.
+      #
+      # @return [Array] cleared array ready for use
+      #
+      def acquire_array
+        @array_pool.acquire
+      end
+
+      # Returns an array to the pool for reuse.
+      #
+      # @param arr [Array] array to release
+      # @return [Boolean] true if pooled, false if discarded
+      #
+      def release_array(arr)
+        @array_pool.release(arr)
+      end
+
+      # Acquires a buffer with minimum capacity from the pool.
+      #
+      # @param size: [Integer] minimum required capacity
+      # @return [Parsanol::Buffer] buffer with capacity >= size
+      #
+      def acquire_buffer(size:)
+        @buffer_pool.acquire(size: size)
+      end
+
+      # Returns a buffer to the pool for reuse.
+      #
+      # @param buf [Parsanol::Buffer] buffer to release
+      # @return [Boolean] true if pooled, false if discarded
+      #
+      def release_buffer(buf)
+        @buffer_pool.release(buf)
+      end
+
+      # Creates a new capture scope for the duration of the block.
+      #
+      # @yield block executed in new scope
+      #
+      def scope
+        captures.push
+        yield
+      ensure
+        captures.pop
+      end
+
+      # Checks if interval-based caching is active.
+      #
+      # @return [Boolean] true if interval caching enabled
+      #
+      def use_tree_memoization?
+        @use_intervals
+      end
+
+      # Queries interval cache for a cached result.
+      #
+      # @param key [Integer] cache key (atom object_id)
+      # @param start_pos [Integer] starting position
+      # @return [Array, nil] cached [values, end_pos] or nil
+      #
+      def query_tree_memo(key, start_pos)
+        return nil unless @use_intervals
+
+        tree = @interval_trees[key]
+        matches = tree.query_overlapping(start_pos, start_pos + 1)
+        found = matches.find { |interval, _| interval[0] == start_pos }
+        found ? found[1] : nil
+      end
+
+      # Stores a result in the interval cache.
+      #
+      # @param key [Integer] cache key
+      # @param start_pos [Integer] start position
+      # @param values [Array] parsed values
+      # @param end_pos [Integer] end position
+      #
+      def store_tree_memo(key, start_pos, values, end_pos)
+        return unless @use_intervals
+
+        @interval_trees[key].insert(start_pos, end_pos, [values, end_pos])
+      end
+
+      # Marks a cut position for aggressive cache eviction.
+      # Called when a cut operator succeeds.
+      #
+      # @param position [Integer] cut position
+      #
+      def cut!(position)
+        @cut_pos = position
+        @memo.delete_if { |pos, _| pos < position }
+      end
+
+      private
+
+      # Lookup cached result (uses object_id for speed)
+      def lookup(atom, pos)
+        @memo[pos][atom.object_id]
+      end
+
+      # Store result in cache
+      def set(atom, pos, val)
+        @memo[pos][atom.object_id] = val
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/context_optimized.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Experimental: Position-based cache eviction for Context
+# Based on PEG theory: in linear parsing, positions behind current position
+# will never be revisited, so we can evict them to reduce memory
+
+module Parsanol
+  module Atoms
+    class Context
+      # Add position tracking for cache eviction
+      attr_reader :current_position
+
+      def try_with_cache(obj, source, consume_all)
+        return obj.try(source, self, consume_all) unless obj.cached?
+
+        key = source.pos
+        @current_position = key
+        atom_cache = @cache[obj]
+
+        # Try to fetch from cache
+        return atom_cache.fetch(key) if atom_cache.key?(key)
+
+        # Cache miss - compute result
+        result = obj.try(source, self, consume_all)
+        atom_cache[key] = result
+
+        # Evict old positions if cache is getting large
+        # Keep only positions within a window of current position
+        if atom_cache.size > 100
+          min_pos = key - 50 # Keep 50 positions behind
+          atom_cache.delete_if { |pos, _| pos < min_pos }
+        end
+
+        result
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/custom.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # Base class for creating custom parser atoms.
+    #
+    # Custom atoms allow extending Parsanol with domain-specific matching logic
+    # that cannot be expressed with the built-in combinators.
+    #
+    # @example Custom atom for matching indentation-sensitive content
+    #   class IndentAtom < Parsanol::Atoms::Custom
+    #     def initialize(expected_indent)
+    #       @expected_indent = expected_indent
+    #       super()
+    #     end
+    #
+    #     # Required: Implement try_match
+    #     def try_match(source, context, consume_all)
+    #       pos = source.pos
+    #       indent = count_indent(source)
+    #
+    #       if indent == @expected_indent
+    #         content = read_until_newline(source)
+    #         [true, content]
+    #       else
+    #         source.pos = pos  # Restore position on failure
+    #         [false, nil]
+    #       end
+    #     end
+    #
+    #     private
+    #
+    #     def count_indent(source)
+    #       # ... implementation ...
+    #     end
+    #   end
+    #
+    #   # Usage in parser
+    #   class MyParser < Parsanol::Parser
+    #     rule(:indented_line) { IndentAtom.new(2) }
+    #   end
+    #
+    class Custom < Base
+      # Required: Implement this method to define matching behavior
+      #
+      # @param source [Parsanol::Source] The input source with position tracking
+      # @param context [Parsanol::Atoms::Context] Parse context for memoization
+      # @param consume_all [Boolean] If true, must consume entire input
+      # @return [Array<Boolean, Object>] Tuple of [success, result]
+      #   - success: true if match succeeded, false otherwise
+      #   - result: matched value on success, nil on failure
+      #
+      # @note You MUST restore source.bytepos on failure for proper backtracking
+      #
+      def try_match(source, context, consume_all)
+        raise NotImplementedError,
+              'Custom atoms must implement #try_match(source, context, consume_all)'
+      end
+
+      # Override of Base#try that delegates to try_match
+      # Handles error reporting and result wrapping
+      #
+      # @api private
+      def try(source, context, consume_all)
+        success, result = try_match(source, context, consume_all)
+
+        if success
+          [true, result]
+        else
+          # Generate error cause for reporting
+          context.err(
+            self,
+            source,
+            "Failed to match custom atom: #{self.class.name}"
+          )
+        end
+      end
+
+      # Optional: Override to provide first set for optimization
+      # Returns the set of characters/strings this atom can match at start
+      #
+      # @return [Set<String>, nil] First set, or nil if not determinable
+      def first_set
+        nil # Unknown by default
+      end
+
+      # Optional: Override to enable caching for this atom
+      # Return false for context-dependent matching (e.g., indentation)
+      #
+      # @return [Boolean] true if atom can be cached
+      def cacheable?
+        true
+      end
+
+      # Optional: Override to provide custom serialization for native parser
+      # Return nil if atom cannot be serialized (must use pure Ruby mode)
+      #
+      # @return [Hash, nil] JSON-serializable representation
+      def to_native_format
+        nil # Not serializable by default
+      end
+
+      # Override to_s_inner for debug printing
+      # @api private
+      def to_s_inner(_prec = nil)
+        "custom(#{self.class.name})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/cut.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# Cut operator for PEG grammars
+#
+# A cut operator (↑) instructs the parser to discard backtrack information
+# at a specific point. This enables more aggressive cache eviction and can
+# reduce space complexity from O(n) to O(1).
+#
+# Reference: Mizushima et al. (2010) "Packrat Parsers Can Handle Practical
+# Grammars in Mostly Constant Space"
+#
+# Example:
+#
+#   rule(:statement) {
+#     str('if').cut >> condition >> then_clause |
+#     str('while').cut >> condition >> body |
+#     str('print').cut >> expression
+#   }
+#
+# After 'if' succeeds, the cut discards backtrack info for 'while' and 'print'.
+# This means if the parse fails later in the 'if' branch, we won't try the
+# other alternatives.
+#
+module Parsanol
+  module Atoms
+    class Cut < Parsanol::Atoms::Base
+      attr_reader :parslet
+
+      def initialize(parslet)
+        super()
+        @parslet = parslet
+      end
+
+      def try(source, context, consume_all)
+        # First, try to match the parslet
+        success, value = parslet.apply(source, context, consume_all)
+
+        return [success, value] unless success
+
+        # On success, signal to context that a cut has occurred
+        # This allows the context to:
+        # 1. Mark the current position as a cut point
+        # 2. Empty the backtrack stack (we won't backtrack past here)
+        # 3. Aggressively evict cache entries before this position
+        context.cut!(source.bytepos) if context.respond_to?(:cut!)
+
+        [success, value]
+      end
+
+      # Cut doesn't need caching - it's a thin wrapper
+      def cached?
+        false
+      end
+
+      def to_s_inner(prec)
+        "#{parslet.to_s(prec)}↑"
+      end
+
+      # FIRST set of cut is same as wrapped parslet
+      # Cut doesn't change matching behavior, only affects backtracking
+      def compute_first_set
+        parslet.first_set
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/dsl.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# Parser composition DSL - chainable methods for building parser atoms.
+# All atoms can use these methods to combine into larger parsers.
+#
+# Inspired by Parslet (MIT License).
+
+module Parsanol
+  module Atoms
+    module DSL
+      # Repeats the current atom between min and max times.
+      # If max is nil, there is no upper limit.
+      #
+      # @example
+      #   str('a').repeat           # match zero or more 'a's
+      #   str('a').repeat(1, 3)   # match 1-3 `a`s
+      def repeat(min = 0, max = nil)
+        Parsanol::Atoms::Repetition.new(self, min, max)
+      end
+
+      # Matches atom optionally (0 or 1 times).
+      # Result is nil if not present, otherwise the matched value.
+      #
+      # @example
+      #   str('foo').maybe   # => nil or 'foo'
+      def maybe
+        Parsanol::Atoms::Repetition.new(self, 0, 1, :maybe)
+      end
+
+      # Ignores the result of a match - returns nil always.
+      #
+      # @example
+      #   str('foo').ignore   # => nil (not 'foo')
+      def ignore
+        Parsanol::Atoms::Ignored.new(self)
+      end
+
+      # Chains two atoms in sequence.
+      #
+      # @example
+      #   str('a') >> str('b')
+      def >>(other)
+        Parsanol::Atoms::Sequence.new(self, other)
+      end
+
+      # Chains two atoms as alternatives (ordered choice).
+      #
+      # @example
+      #   str('a') | str('b')   # matches 'a' or `b`
+      def |(other)
+        Parsanol::Atoms::Alternative.new(self, other)
+      end
+
+      # Negative lookahead - succeeds only if atom is absent.
+      #
+      # @example
+      #   str('a').absent?
+      def absent?
+        Parsanol::Atoms::Lookahead.new(self, false)
+      end
+
+      # Positive lookahead - succeeds only if atom is present.
+      #
+      # @example
+      #   str('a').present?
+      def present?
+        Parsanol::Atoms::Lookahead.new(self, true)
+      end
+
+      # Labels a match for tree output.
+      #
+      # @example
+      #   str('a').as(:b)   # => {:b => 'a'}
+      def as(name)
+        Parsanol::Atoms::Named.new(self, name)
+      end
+
+      # Captures match result for later reference.
+      #
+      # @example
+      #   str('a').capture(:first) >> dynamic { str(ctx.captures[:first]) }
+      def capture(name)
+        Parsanol::Atoms::Capture.new(self, name)
+      end
+
+      # Commit point - prevents backtracking after successful match.
+      # Use with caution: cuts prevent backtracking to alternatives.
+      #
+      # @example
+      #   str('if').cut >> condition >> body |
+      def cut
+        Parsanol::Atoms::Cut.new(self)
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/dynamic.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Evaluates a block at parse time. The result from the block must be a parser
+# (something which implements #apply). In the first case, the parser will then
+# be applied to the input, creating the result.
+#
+# Dynamic parses are never cached.
+#
+# Example:
+#   dynamic { rand < 0.5 ? str('a') : str('b') }
+#
+module Parsanol
+  module Atoms
+    class Dynamic < Parsanol::Atoms::Base
+      attr_reader :block
+
+      def initialize(block)
+        @block = block
+      end
+
+      def cached?
+        false
+      end
+
+      def try(source, context, consume_all)
+        # Phase 55: Cache @block ivar to reduce lookup overhead
+        block = @block
+        result = block.call(source, context)
+
+        # Result is a parslet atom.
+        result.apply(source, context, consume_all)
+      end
+
+      def to_s_inner(_prec)
+        'dynamic { ... }'
+      end
+    end
+  end
+end