parsanol 1.3.13-arm-linux

data/lib/parsanol/atoms/can_flatten.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # A series of helper functions that have the common topic of flattening
+    # result values into the intermediary tree that consists of Ruby Hashes and
+    # Arrays.
+    #
+    # This module has one main function, #flatten, that takes an annotated
+    # structure as input and returns the reduced form that users expect from
+    # Atom#parse.
+    #
+    # NOTE: Since all of these functions are just that, functions without
+    # side effects, they are in a module and not in a class. Its hard to draw
+    # the line sometimes, but this is beyond.
+    #
+    module CanFlatten
+      # Takes a mixed value coming out of a parslet and converts it to a return
+      # value for the user by dropping things and merging hashes.
+      #
+      # Named is set to true if this result will be embedded in a Hash result from
+      # naming something using <code>.as(...)</code>. It changes the folding
+      # semantics of repetition.
+      #
+      def flatten(value, named = false)
+        # Passes through everything that isn't an array of things
+        # Phase 43: Use simpler check - if it's not an Array, return as-is
+        return value unless value.is_a?(Array)
+
+        # Extracts the s-expression tag
+        tag = value[0]
+
+        # Phase 43: Optimize flattening - reduce method call overhead
+        # For single element arrays (common case), handle directly
+        tail_size = value.size - 1
+        if tail_size == 1
+          flattened = flatten(value[1])
+          case tag
+          when :sequence
+            return flattened
+          when :maybe
+            return named ? flattened : (flattened || "")
+          when :repetition
+            return flatten_repetition([flattened], named)
+          end
+        end
+
+        # Flatten each element
+        result = Array.new(tail_size)
+        i = 0
+        while i < tail_size
+          result[i] = flatten(value[i + 1])
+          i += 1
+        end
+
+        case tag
+        when :sequence
+          return flatten_sequence(result)
+        when :maybe
+          return named ? result.first : result.first || ""
+        when :repetition
+          return flatten_repetition(result, named)
+        end
+
+        raise "BUG: Unknown tag #{tag.inspect}."
+      end
+
+      # Lisp style fold left where the first element builds the basis for
+      # an inject. Optimized with early return and reduced method calls.
+      #
+      def foldl(list)
+        len = list.size
+        return "" if len.zero?
+        return list[0] if len == 1 # Fast path for single element
+
+        result = list[0]
+        i = 1
+        while i < len
+          result = yield(result, list[i])
+          i += 1
+        end
+        result
+      end
+
+      # Flatten results from a sequence of parslets.
+      #
+      # @api private
+      #
+      def flatten_sequence(list)
+        foldl(list.compact) do |r, e| # and then merge flat elements
+          merge_fold(r, e)
+        end
+      end
+
+      # @api private
+      # Phase 43: Optimized merge_fold - reduce repeated class checks
+      def merge_fold(l, r)
+        l_class = l.class
+        r_class = r.class
+
+        # equal pairs: merge. ----------------------------------------------------
+        if l_class == r_class
+          return l + r unless l_class == Hash
+
+          warn_about_duplicate_keys(l, r)
+          return l.merge(r)
+
+        end
+
+        # Phase 43: Cache instance_of? checks to avoid repeated method calls
+        # unequal pairs: hoist to same level. ------------------------------------
+        l_is_slice = l.instance_of?(Parsanol::Slice)
+        r_is_slice = r.instance_of?(Parsanol::Slice)
+        l_is_str = l_class == String || l_is_slice
+        r_is_str = r_class == String || r_is_slice
+
+        # Maybe classes are not equal, but both are stringlike?
+        if l_is_str && r_is_str
+          # if we're merging a String with a Slice, the slice wins.
+          return r if r_is_slice
+          return l if l_is_slice
+
+          raise "NOTREACHED: What other stringlike classes are there?"
+        end
+
+        # special case: If one of them is a string/slice, the other is more important
+        return l if r_is_str
+        return r if l_is_str
+
+        # otherwise just create an array for one of them to live in
+        return l + [r] if r_class == Hash
+        return [l] + r if l_class == Hash
+
+        raise "Unhandled case when foldr'ing sequence."
+      end
+
+      # Flatten results from a repetition of a single parslet. named indicates
+      # whether the user has named the result or not. If the user has named
+      # the results, we want to leave an empty list alone - otherwise it is
+      # turned into an empty string.
+      #
+      # @api private
+      #
+      # Phase 43: Optimized flatten_repetition - reduce array iterations
+      def flatten_repetition(list, named)
+        # Phase 43: Single pass to check for hashes and arrays
+        has_hash = false
+        has_array = false
+
+        i = 0
+        len = list.size
+        while i < len
+          e = list[i]
+          has_hash = true if e.instance_of?(Hash)
+          has_array = true if e.instance_of?(Array)
+          break if has_hash && has_array # Early exit if both found
+
+          i += 1
+        end
+
+        if has_hash
+          # If keyed subtrees are in the array, we'll want to discard all
+          # strings inbetween. To keep them, name them.
+          return list.select { |e| e.instance_of?(Hash) }
+        end
+
+        if has_array
+          # If any arrays are nested in this array, flatten all arrays to this
+          # level.
+          return list
+              .select { |e| e.instance_of?(Array) }
+              .flatten(1)
+        end
+
+        # Consistent handling of empty lists, when we act on a named result
+        return [] if named && list.empty?
+
+        # If there are only strings, concatenate them and return that.
+        foldl(list.compact) { |s, e| s + e }
+      end
+
+      # That annoying warning 'Duplicate subtrees while merging result' comes
+      # from here. You should add more '.as(...)' names to your intermediary tree.
+      #
+      def warn_about_duplicate_keys(h1, h2)
+        d = h1.keys & h2.keys
+        return if d.empty?
+
+        warn "Duplicate subtrees while merging result of \n  #{inspect}\nonly the values " \
+             "of the latter will be kept. (keys: #{d.inspect})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/capture.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Captures the result of parsing and stores it for later use.
+# Use the capture method to capture a sub-expression result, then
+# access it via context.captures[:name] in dynamic blocks.
+#
+# @example
+#   str('a').capture(:first) >> dynamic { |ctx| str(ctx.captures[:first]) }
+#
+module Parsanol
+  module Atoms
+    class Capture < Parsanol::Atoms::Base
+      attr_reader :inner_atom, :capture_key
+
+      def initialize(atom, name)
+        super()
+        @inner_atom = atom
+        @capture_key = name.to_sym
+      end
+
+      def apply(source, context, consume_all)
+        success, result = @inner_atom.apply(source, context, consume_all)
+
+        if success
+          # Flatten and store the captured value in context
+          flattened = flatten(result)
+          context.captures[@capture_key] = flattened
+        end
+
+        [success, result]
+      end
+
+      def to_s_inner(prec)
+        "(#{@capture_key.inspect} = #{@inner_atom.to_s(prec)})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/context.rb

@@ -0,0 +1,357 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # Parsing context that coordinates memoization caching, error reporting,
+    # and resource pooling. Created fresh for each parse operation.
+    #
+    # Key responsibilities:
+    # - Packrat-style memoization (caching parse results by position+atom)
+    # - Pluggable error reporting through reporter interface
+    # - Object pooling for arrays and buffers to reduce GC pressure
+    # - Adaptive caching based on input size
+    #
+    # @example Basic usage
+    #   ctx = Context.new(reporter)
+    #   result = ctx.try_with_cache(parser, source, true)
+    #
+    # Inspired by packrat parsing memoization and incremental parsing techniques.
+    #
+    class Context
+      # Per-parser cache size thresholds based on profiling different grammar types
+      # Different grammars benefit from caching at different input sizes
+      PARSER_CACHE_LIMITS = {
+        "JsonParser" => 10_000,      # JSON needs large inputs to benefit
+        "ErbParser" => 800,          # ERB benefits earlier
+        "CalcParser" => 2000,        # Calculator has low repetition
+        "SentenceParser" => 5000,    # Linear grammar, minimal benefit
+        :default => 1000,
+      }.freeze
+
+      # Number of observed backtrack events before packrat caching engages.
+      BACKTRACK_ACTIVATION_LIMIT = 64
+
+      # Creates a new parsing context.
+      #
+      # @param error_reporter [#err, #err_at] error reporter instance
+      # @param interval_cache: [Boolean] enable GPeg-style interval caching
+      # @param adaptive_cache_threshold: [Integer, nil] minimum input size for caching
+      # @param parser_class: [Class, nil] parser class for threshold selection
+      #
+      def initialize(error_reporter = Parsanol::ErrorReporter::Tree.new,
+                     interval_cache: false,
+                     adaptive_cache_threshold: nil,
+                     parser_class: nil)
+        # Core memoization cache: position -> { atom_id -> [result, advance] }
+        @memo = {}
+
+        # Error reporting delegate
+        @reporter = error_reporter
+
+        # Capture scope for variable bindings
+        @captures = Parsanol::Scope.new
+
+        # Cache eviction state
+        @furthest_pos = 0
+        @evict_threshold = 200
+        @evict_counter = 0
+        @evict_interval = 100
+
+        # Object pools for reducing allocations
+        @array_pool = Parsanol::Pools::ArrayPool.new(size: 10_000, preallocate: false)
+        @buffer_pool = Parsanol::Pools::BufferPool.new(pool_size: 100)
+
+        # Selective memoization tracking
+        @hit_stats = Hash.new(0)
+        @miss_stats = Hash.new(0)
+        @min_hits_for_cache = 2
+
+        # Optional GPeg-style interval caching
+        @use_intervals = interval_cache
+        if @use_intervals
+          require "parsanol/interval_tree"
+          require "parsanol/edit_tracker"
+          @interval_trees = Hash.new { |h, k| h[k] = Parsanol::IntervalTree.new }
+          @edits = Parsanol::EditTracker.new
+        end
+
+        # Cut operator support for aggressive eviction
+        @cut_pos = 0
+
+        # Determine adaptive cache threshold
+        threshold = adaptive_cache_threshold
+        if threshold.nil? && parser_class
+          name = parser_class.name&.split("::")&.last
+          threshold = PARSER_CACHE_LIMITS[name] || PARSER_CACHE_LIMITS[:default]
+        end
+        threshold ||= PARSER_CACHE_LIMITS[:default]
+
+        @adaptive_threshold = threshold
+        @input_len = nil
+        @caching_active = false
+        @backtrack_events = 0
+      end
+
+      # Attempts to parse using memoization. Returns cached result if available,
+      # otherwise executes the parser and caches the result.
+      #
+      # @param atom [Parsanol::Atoms::Base] parser to apply
+      # @param src [Parsanol::Source] input source
+      # @param must_consume_all [Boolean] require complete consumption
+      # @return [Array(Boolean, Object)] parse result tuple
+      #
+      def try_with_cache(atom, src, must_consume_all)
+        # Skip caching for atoms that don't benefit from it
+        return atom.try(src, self, must_consume_all) unless atom.cached?
+
+        # Use interval-based caching if enabled
+        return try_with_interval(atom, src, must_consume_all) if @use_intervals
+
+        # Adaptive activation: packrat memoization costs more than it saves
+        # on deterministic forward-only grammars, so start uncached and only
+        # engage once real backtracking (re-parsing behind the progress
+        # frontier) is observed.
+        return try_uncached_probe(atom, src, must_consume_all) unless @caching_active
+
+        pos = src.bytepos
+        key = atom.object_id
+        entry = @memo[pos]
+
+        # Periodic cache eviction to prevent unbounded growth
+        if pos > @furthest_pos
+          @furthest_pos = pos
+          @evict_counter += 1
+
+          if @evict_counter >= @evict_interval
+            @evict_counter = 0
+            cutoff = pos - @evict_threshold
+            @memo.delete_if { |p, _| p < cutoff }
+          end
+        end
+
+        # Check for cache hit (avoid default-block Hash allocation per probe)
+        if entry&.key?(key)
+          @hit_stats[key] += 1
+          outcome, delta = entry[key]
+          src.bytepos = pos + delta
+          return outcome
+        end
+
+        # Cache miss - execute and store
+        @miss_stats[key] += 1
+        outcome = atom.try(src, self, must_consume_all)
+        delta = src.bytepos - pos
+
+        # Only cache if beneficial (heuristic)
+        attempts = @hit_stats[key] + @miss_stats[key]
+        if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
+          (@memo[pos] ||= {})[key] =
+            [outcome,
+             delta]
+        end
+
+        outcome
+      end
+
+      # GPeg-style interval-based caching for incremental parsing.
+      #
+      # @param atom [Parsanol::Atoms::Base] parser to apply
+      # @param src [Parsanol::Source] input source
+      # @param must_consume_all [Boolean] require complete consumption
+      # @return [Array(Boolean, Object)] parse result tuple
+      #
+      def try_with_interval(atom, src, must_consume_all)
+        pos = src.bytepos
+        key = atom.object_id
+
+        tree = @interval_trees[key]
+        cached = tree.query_exact(pos, pos)
+
+        if cached
+          @hit_stats[key] += 1
+          outcome, delta = cached
+          src.bytepos = pos + delta
+          return outcome
+        end
+
+        @miss_stats[key] += 1
+        outcome = atom.try(src, self, must_consume_all)
+        delta = src.bytepos - pos
+        end_pos = pos + delta
+
+        attempts = @hit_stats[key] + @miss_stats[key]
+        if attempts <= @min_hits_for_cache || @hit_stats[key].positive?
+          tree.insert(pos, end_pos,
+                      [outcome, delta])
+        end
+
+        outcome
+      end
+
+      # Pre-allocated result constants
+      SUCCESS_RESULT = [true, nil].freeze
+      ERROR_RESULT = [false, nil].freeze
+
+      # Reports an error at a specific position.
+      #
+      # @return [Array(Boolean, Object)] error result tuple
+      #
+      def err_at(*)
+        return [false, @reporter.err_at(*)] if @reporter
+
+        ERROR_RESULT
+      end
+
+      # Reports an error at the current position.
+      #
+      # @return [Array(Boolean, Object)] error result tuple
+      #
+      def err(*)
+        return [false, @reporter.err(*)] if @reporter
+
+        ERROR_RESULT
+      end
+
+      # Reports a successful parse.
+      #
+      # @return [Array(Boolean, Object)] success result tuple
+      #
+      def succ(*)
+        return SUCCESS_RESULT unless @reporter
+
+        val = @reporter.succ(*)
+        return SUCCESS_RESULT if val.nil?
+
+        [true, val]
+      end
+
+      # @return [Parsanol::Scope] capture variable bindings
+      attr_reader :captures
+
+      # @return [Parsanol::Pools::ArrayPool] array object pool
+      attr_reader :array_pool
+
+      # @return [Parsanol::Pools::BufferPool] buffer object pool
+      attr_reader :buffer_pool
+
+      # Acquires an empty array from the pool.
+      #
+      # @return [Array] cleared array ready for use
+      #
+      def acquire_array
+        @array_pool.acquire
+      end
+
+      # Returns an array to the pool for reuse.
+      #
+      # @param arr [Array] array to release
+      # @return [Boolean] true if pooled, false if discarded
+      #
+      def release_array(arr)
+        @array_pool.release(arr)
+      end
+
+      # Acquires a buffer with minimum capacity from the pool.
+      #
+      # @param size: [Integer] minimum required capacity
+      # @return [Parsanol::Buffer] buffer with capacity >= size
+      #
+      def acquire_buffer(size:)
+        @buffer_pool.acquire(size: size)
+      end
+
+      # Returns a buffer to the pool for reuse.
+      #
+      # @param buf [Parsanol::Buffer] buffer to release
+      # @return [Boolean] true if pooled, false if discarded
+      #
+      def release_buffer(buf)
+        @buffer_pool.release(buf)
+      end
+
+      # Creates a new capture scope for the duration of the block.
+      #
+      # @yield block executed in new scope
+      #
+      def scope
+        captures.push
+        yield
+      ensure
+        captures.pop
+      end
+
+      # Checks if interval-based caching is active.
+      #
+      # @return [Boolean] true if interval caching enabled
+      #
+      def use_tree_memoization?
+        @use_intervals
+      end
+
+      # Queries interval cache for a cached result.
+      #
+      # @param key [Integer] cache key (atom object_id)
+      # @param start_pos [Integer] starting position
+      # @return [Array, nil] cached [values, end_pos] or nil
+      #
+      def query_tree_memo(key, start_pos)
+        return nil unless @use_intervals
+
+        tree = @interval_trees[key]
+        matches = tree.query_overlapping(start_pos, start_pos + 1)
+        found = matches.find { |interval, _| interval[0] == start_pos }
+        found ? found[1] : nil
+      end
+
+      # Stores a result in the interval cache.
+      #
+      # @param key [Integer] cache key
+      # @param start_pos [Integer] start position
+      # @param values [Array] parsed values
+      # @param end_pos [Integer] end position
+      #
+      def store_tree_memo(key, start_pos, values, end_pos)
+        return unless @use_intervals
+
+        @interval_trees[key].insert(start_pos, end_pos, [values, end_pos])
+      end
+
+      # Marks a cut position for aggressive cache eviction.
+      # Called when a cut operator succeeds.
+      #
+      # @param position [Integer] cut position
+      #
+      def cut!(position)
+        @cut_pos = position
+        @memo.delete_if { |pos, _| pos < position }
+      end
+
+      private
+
+      # Executes an atom without memoization while watching for backtracking.
+      # A failed attempt at a position behind the progress frontier means work
+      # is being re-done; once that repeats, packrat caching engages.
+      def try_uncached_probe(atom, src, must_consume_all)
+        pos = src.bytepos
+        outcome = atom.try(src, self, must_consume_all)
+        if pos > @furthest_pos
+          @furthest_pos = pos
+        elsif !outcome[0] && pos < @furthest_pos
+          @backtrack_events += 1
+          @caching_active = true if @backtrack_events >= BACKTRACK_ACTIVATION_LIMIT
+        end
+        outcome
+      end
+
+      # Lookup cached result (uses object_id for speed)
+      def lookup(atom, pos)
+        @memo[pos][atom.object_id]
+      end
+
+      # Store result in cache
+      def set(atom, pos, val)
+        @memo[pos][atom.object_id] = val
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/context_optimized.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Experimental: Position-based cache eviction for Context
+# Based on PEG theory: in linear parsing, positions behind current position
+# will never be revisited, so we can evict them to reduce memory
+
+module Parsanol
+  module Atoms
+    class Context
+      # Add position tracking for cache eviction
+      attr_reader :current_position
+
+      def try_with_cache(obj, source, consume_all)
+        return obj.try(source, self, consume_all) unless obj.cached?
+
+        key = source.pos
+        @current_position = key
+        atom_cache = @cache[obj]
+
+        # Try to fetch from cache
+        return atom_cache.fetch(key) if atom_cache.key?(key)
+
+        # Cache miss - compute result
+        result = obj.try(source, self, consume_all)
+        atom_cache[key] = result
+
+        # Evict old positions if cache is getting large
+        # Keep only positions within a window of current position
+        if atom_cache.size > 100
+          min_pos = key - 50 # Keep 50 positions behind
+          atom_cache.delete_if { |pos, _| pos < min_pos }
+        end
+
+        result
+      end
+    end
+  end
+end