parsanol 3.0.0

data/lib/parsanol/atoms/dynamic.rb

@@ -0,0 +1,33 @@
+# Evaluates a block at parse time. The result from the block must be a parser
+# (something which implements #apply). In the first case, the parser will then
+# be applied to the input, creating the result.
+#
+# Dynamic parses are never cached.
+#
+# Example:
+#   dynamic { rand < 0.5 ? str('a') : str('b') }
+#
+class Parsanol::Atoms::Dynamic < Parsanol::Atoms::Base
+  attr_reader :block
+
+  def initialize(block)
+    @block = block
+  end
+
+  def cached?
+    false
+  end
+
+  def try(source, context, consume_all)
+    # Phase 55: Cache @block ivar to reduce lookup overhead
+    block = @block
+    result = block.call(source, context)
+
+    # Result is a parslet atom.
+    return result.apply(source, context, consume_all)
+  end
+
+  def to_s_inner(prec)
+    "dynamic { ... }"
+  end
+end

data/lib/parsanol/atoms/entity.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This wraps pieces of parslet definition and gives them a name. The wrapped
+# piece is lazily evaluated and cached. This has two purposes:
+#
+# * Avoid infinite recursion during evaluation of the definition
+# * Be able to print things by their name, not by their sometimes
+#   complicated content.
+#
+# You don't normally use this directly, instead you should generate it by
+# using the structuring method Parslet.rule.
+#
+class Parsanol::Atoms::Entity < Parsanol::Atoms::Base
+  attr_reader :name, :block
+  def initialize(name, label=nil, &block)
+    super()
+
+    @name = name
+    @label = label
+    @block = block
+    @parslet = nil
+  end
+
+  def try(source, context, consume_all)
+    # Phase 54: Cache parslet method result to reduce method call overhead
+    p = parslet
+    p.apply(source, context, consume_all)
+  end
+
+  # Entity is just a thin wrapper that delegates to the underlying parslet.
+  # The underlying parslet is already cached, so caching the wrapper is redundant.
+  def cached?
+    false
+  end
+
+  def parslet
+    return @parslet unless @parslet.nil?
+    @parslet = @block.call
+    raise_not_implemented if @parslet.nil?
+    @parslet.label = @label
+    @parslet
+  end
+
+  def to_s_inner(prec)
+    name.to_s.upcase
+  end
+private
+  def raise_not_implemented
+    trace = caller.reject {|l| l =~ %r{#{Regexp.escape(__FILE__)}}} # blatantly stolen from dependencies.rb in activesupport
+    exception = NotImplementedError.new("rule(#{name.inspect}) { ... }  returns nil. Still not implemented, but already used?")
+    exception.set_backtrace(trace)
+
+    raise exception
+  end
+end

data/lib/parsanol/atoms/ignored.rb

@@ -0,0 +1,28 @@
+# Ignores the result of a match.
+#
+# Example:
+#
+#   str('foo')            # will return 'foo',
+#   str('foo').ignore     # will return nil
+#
+class Parsanol::Atoms::Ignored < Parsanol::Atoms::Base
+  attr_reader :parslet
+  def initialize(parslet)
+    super()
+
+    @parslet = parslet
+  end
+
+  def apply(source, context, consume_all)
+    # Phase 55: Cache @parslet ivar to reduce lookup overhead
+    parslet = @parslet
+    success, _ = result = parslet.apply(source, context, consume_all)
+
+    return result unless success
+    succ(nil)
+  end
+
+  def to_s_inner(prec)
+    "ignored(#{parslet.to_s(prec)})"
+  end
+end

data/lib/parsanol/atoms/infix.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+class Parsanol::Atoms::Infix < Parsanol::Atoms::Base
+  attr_reader :element, :operations, :reducer
+
+  def initialize(element, operations, &reducer)
+    super()
+
+    @element = element
+    @operations = operations
+    @reducer = reducer || lambda { |left, op, right| {l: left, o: op, r: right} }
+  end
+
+  def try(source, context, consume_all)
+    return catch(:error) {
+      return succ(
+        produce_tree(
+          precedence_climb(source, context, consume_all)))
+    }
+  end
+
+  # Turns an array of the form ['1', '+', ['2', '*', '3']] into a hash that
+  # reflects the same structure.
+  #
+  def produce_tree(ary)
+    return ary unless ary.kind_of? Array
+
+    # Phase 55: Cache @reducer ivar to reduce lookup overhead in loop
+    reducer = @reducer
+    left = ary.shift
+
+    until ary.empty?
+      op, right = ary.shift(2)
+
+      # p [left, op, right]
+
+      if right.kind_of? Array
+        # Subexpression -> Subhash
+        left = reducer.call(left, op, produce_tree(right))
+      else
+        left = reducer.call(left, op, right)
+      end
+    end
+
+    left
+  end
+
+  # A precedence climbing algorithm married to parslet, as described here
+  #   http://eli.thegreenplace.net/2012/08/02/parsing-expressions-by-precedence-climbing/
+  #
+  # @note Error handling in this routine is done by throwing :error and
+  #       as a value the error to return to parslet. This avoids cluttering
+  #       the recursion logic here with parslet error handling.
+  #
+  def precedence_climb(source, context, consume_all, current_prec=1, needs_element=false)
+    # Phase 55: Cache @element ivar to reduce lookup overhead
+    element = @element
+    result = []
+
+    # To even begin parsing an arithmetic expression, there needs to be
+    # at least one @element.
+    success, value = element.apply(source, context, false)
+
+    unless success
+      throw :error, context.err(self, source, "#{element.inspect} was expected", [value])
+    end
+
+    result << flatten(value, true)
+
+    # Loop until we fail on operator matching or until input runs out.
+    loop do
+      op_pos = source.bytepos
+      op_match, prec, assoc = match_operation(source, context, false)
+
+      # If no operator could be matched here, one of several cases
+      # applies:
+      #
+      # - end of file
+      # - end of expression
+      # - syntax error
+      #
+      # We abort matching the expression here.
+      break unless op_match
+
+      if prec >= current_prec
+        # Inline: next_prec = (assoc == :left) ? prec+1 : prec
+        result << op_match
+        result << precedence_climb(
+          source, context, consume_all,
+          (assoc == :left) ? prec+1 : prec, true)
+      else
+        source.bytepos = op_pos
+        # Inline unwrap
+        return result.size == 1 ? result.first : result
+      end
+    end
+
+    # Inline unwrap
+    return result.size == 1 ? result.first : result
+  end
+
+  def match_operation(source, context, consume_all)
+    # Phase 55: Cache @operations ivar to reduce lookup overhead in loop
+    operations = @operations
+    errors = []
+    operations.each do |op_atom, prec, assoc|
+      success, value = op_atom.apply(source, context, consume_all)
+      return flatten(value, true), prec, assoc if success
+
+      # assert: this was in fact an error, accumulate
+      errors << value
+    end
+
+    return nil
+  end
+
+  def to_s_inner(prec)
+    ops = @operations.map { |o, _, _| o.inspect }.join(', ')
+    "infix_expression(#{@element.inspect}, [#{ops}])"
+  end
+end

data/lib/parsanol/atoms/lookahead.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# Either positive or negative lookahead, doesn't consume its input.
+#
+# Example:
+#
+#   str('foo').present? # matches when the input contains 'foo', but leaves it
+#
+class Parsanol::Atoms::Lookahead < Parsanol::Atoms::Base
+  attr_reader :positive
+  attr_reader :bound_parslet
+
+  def initialize(bound_parslet, positive=true)
+    super()
+
+    # Model positive and negative lookahead by testing this flag.
+    @positive = positive
+    @bound_parslet = bound_parslet
+
+    # Phase 60: Pre-compute and freeze error messages
+    @error_msgs = {
+      :positive => ["Input should start with ".freeze, bound_parslet].freeze,
+      :negative => ["Input should not start with ".freeze, bound_parslet].freeze
+    }.freeze
+  end
+
+  def try(source, context, consume_all)
+    # Phase 55: Cache ivars to reduce lookup overhead
+    positive = @positive
+    bound_parslet = @bound_parslet
+
+    # Phase 23: Lookahead position restore optimization
+    # Always restore position after lookahead, simplify logic
+    rewind_pos = source.bytepos
+
+    success, _ = bound_parslet.apply(source, context, consume_all)
+
+    # Always restore position - lookahead never consumes input
+    source.bytepos = rewind_pos
+
+    # Positive lookahead: success when parslet matches
+    return succ(nil) if positive && success
+    return context.err_at(self, source, @error_msgs[:positive], source.bytepos) if positive
+
+    # Negative lookahead: success when parslet fails
+    return context.err_at(self, source, @error_msgs[:negative], source.bytepos) if success
+    return succ(nil)
+  end
+
+  precedence LOOKAHEAD
+  def to_s_inner(prec)
+    @char = positive ? '&' : '!'
+
+    "#{@char}#{bound_parslet.to_s(prec)}"
+  end
+
+  # FIRST set of lookahead:
+  # - Positive lookahead (&e): FIRST is EPSILON (doesn't consume)
+  # - Negative lookahead (!e): FIRST is EPSILON (doesn't consume)
+  # Both always match without consuming, so they can match empty
+  def compute_first_set
+    Set.new([Parsanol::FirstSet::EPSILON])
+  end
+end

data/lib/parsanol/atoms/named.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Names a match to influence tree construction.
+#
+# Example:
+#
+#   str('foo')            # will return 'foo',
+#   str('foo').as(:foo)   # will return :foo => 'foo'
+#
+class Parsanol::Atoms::Named < Parsanol::Atoms::Base
+  attr_reader :parslet, :name
+  def initialize(parslet, name)
+    super()
+
+    @parslet, @name = parslet, name
+  end
+
+  def apply(source, context, consume_all)
+    # Phase 52: Cache @parslet ivar to reduce lookup overhead
+    parslet = @parslet
+
+    success, value = result = parslet.apply(source, context, consume_all)
+
+    return result unless success
+    succ(
+      produce_return_value(
+        value))
+  end
+
+  # Named is just a thin wrapper that delegates to the underlying parslet.
+  # The underlying parslet is already cached, so caching the wrapper is redundant.
+  def cached?
+    false
+  end
+
+  def to_s_inner(prec)
+    "#{name}:#{parslet.to_s(prec)}"
+  end
+
+  # FIRST set of named atom is same as wrapped parslet
+  # Named is just a wrapper that doesn't change matching behavior
+  def compute_first_set
+    parslet.first_set
+  end
+
+private
+  def produce_return_value(val)
+    { name => flatten(val, true) }
+  end
+end

data/lib/parsanol/atoms/re.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Matches a special kind of regular expression that only ever matches one
+# character at a time. Useful members of this family are: <code>character
+# ranges, \\w, \\d, \\r, \\n, ...</code>
+#
+# Example:
+#
+#   match('[a-z]')  # matches a-z
+#   match('\s')     # like regexps: matches space characters
+#
+class Parsanol::Atoms::Re < Parsanol::Atoms::Base
+  attr_reader :match, :re
+  def initialize(match)
+    super()
+
+    @match = match.to_s
+    @re    = Regexp.new(self.match, Regexp::MULTILINE)
+
+    # Phase 60: Pre-compute and freeze error messages
+    @error_msgs = {
+      premature: 'Premature end of input'.freeze,
+      failed: "Failed to match #{match.inspect[1..-2]}".freeze
+    }.freeze
+  end
+
+  def try(source, context, consume_all)
+    # Phase 55: Cache @re ivar to reduce lookup overhead
+    re = @re
+    return succ(source.consume(1)) if source.matches?(re)
+
+    # No string could be read
+    return context.err(self, source, @error_msgs[:premature]) \
+      if source.chars_left < 1
+
+    # No match
+    return context.err(self, source, @error_msgs[:failed])
+  end
+
+  def to_s_inner(prec)
+    match.inspect[1..-2]
+  end
+
+  # Regex matching is already very fast (single character match).
+  # Caching adds overhead without benefit for such simple operations.
+  def cached?
+    false
+  end
+
+  # Session 13: Re always produces flat results (Parsanol::Slice)
+  # No nested structures, so flatten can skip processing
+  def flat?
+    true
+  end
+
+  # FIRST set for Re is the regex itself
+  # Conservative: could theoretically analyze regex to extract literal prefixes
+  def compute_first_set
+    Set.new([self])
+  end
+end

data/lib/parsanol/atoms/repetition.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+
+# Matches a parslet repeatedly.
+#
+# Example:
+#
+#   str('a').repeat(1,3)  # matches 'a' at least once, but at most three times
+#   str('a').maybe        # matches 'a' if it is present in the input (repeat(0,1))
+#
+class Parsanol::Atoms::Repetition < Parsanol::Atoms::Base
+  attr_reader :min, :max, :parslet
+  def initialize(parslet, min, max, tag=:repetition)
+    super()
+
+    raise ArgumentError,
+      "Asking for zero repetitions of a parslet. (#{parslet.inspect} repeating #{min},#{max})" \
+      if max == 0
+
+    @parslet = parslet
+    @min = min
+    @max = max
+    @tag = tag
+
+    # Phase 58: Pre-compute and freeze error messages to avoid allocations
+    @error_msgs = {
+      minrep: "Expected at least #{min} of #{parslet.inspect}".freeze,
+      unconsumed: 'Extra input after last repetition'.freeze
+    }.freeze
+  end
+
+  def error_msgs
+    @error_msgs
+  end
+
+  def try(source, context, consume_all)
+    # Phase 54: Cache ivars to reduce lookup overhead in hot method
+    parslet = @parslet
+    min = @min
+    max = @max
+    tag = @tag
+
+    # Use tree memoization if interval cache is enabled
+    if context.respond_to?(:use_tree_memoization?) && context.use_tree_memoization?
+      return try_with_tree_memoization(source, context, consume_all)
+    end
+
+    # Fast path for .maybe (min=0, max=1) - very common case
+    if min == 0 && max == 1
+      success, value = parslet.apply(source, context, false)
+      return succ([tag, value]) if success
+      # Phase 57b: Use frozen constant for empty repetition array
+      return succ(tag == :repetition ? Parsanol::Atoms::Base::EMPTY_REPETITION_ARRAY : [tag])
+    end
+
+    # Fast path for exact count (min == max)
+    if min == max && max && max <= 3
+      case max
+      when 1
+        success, value = parslet.apply(source, context, consume_all)
+        return success ? succ([tag, value]) : context.err_at(self, source, error_msgs[:minrep], source.bytepos, [value])
+      when 2
+        success, v1 = parslet.apply(source, context, false)
+        return context.err_at(self, source, error_msgs[:minrep], source.bytepos, [v1]) unless success
+        success, v2 = parslet.apply(source, context, consume_all)
+        return success ? succ([tag, v1, v2]) : context.err_at(self, source, error_msgs[:minrep], source.bytepos, [v2])
+      when 3
+        success, v1 = parslet.apply(source, context, false)
+        return context.err_at(self, source, error_msgs[:minrep], source.bytepos, [v1]) unless success
+        success, v2 = parslet.apply(source, context, false)
+        return context.err_at(self, source, error_msgs[:minrep], source.bytepos, [v2]) unless success
+        success, v3 = parslet.apply(source, context, consume_all)
+        return success ? succ([tag, v1, v2, v3]) : context.err_at(self, source, error_msgs[:minrep], source.bytepos, [v3])
+      end
+    end
+
+    # General case for variable or large repetitions
+    try_repetition_general(source, context, consume_all)
+  end
+
+  # GPeg-style tree memoization for repetitions
+  # Caches arrays of successful matches to reuse parsed prefixes
+  def try_with_tree_memoization(source, context, consume_all)
+    start_pos = source.bytepos
+    cache_key = object_id
+
+    # Check if we have a cached tree result at this position
+    cached = context.query_tree_memo(cache_key, start_pos)
+    if cached
+      values, end_pos = cached
+      source.bytepos = end_pos
+      return succ([@tag] + values)
+    end
+
+    # Parse repetition and collect all successful matches
+    occ = 0
+    # Phase 2.2: Use buffer pool for lazy materialization
+    estimated_size = [max || 10, 10].min
+    buffer = context.acquire_buffer(size: estimated_size + 1)  # +1 for tag
+    buffer.push(@tag)  # Add tag first to maintain [:repetition, ...] structure
+    # Positions array is stored in tree memo cache, so use array pool
+    positions = context.acquire_array
+    positions << start_pos  # Track position after each match
+    break_on = nil
+
+    loop do
+      pos_before = source.bytepos
+      success, value = parslet.apply(source, context, false)
+
+      break_on = value
+      break unless success
+
+      occ += 1
+      buffer.push(value)
+      positions << source.bytepos
+
+      # Check max bound
+      break if max && occ >= max
+    end
+
+    # Store tree memo: cache the array of successful matches
+    # This allows reusing the parsed prefix on subsequent parses
+    if occ > 0
+      end_pos = positions[occ]
+      # Must materialize for tree memo storage
+      context.store_tree_memo(cache_key, start_pos, buffer.to_a[1..-1], end_pos)
+    end
+
+    # Check min bound
+    if occ < min
+      source.bytepos = start_pos
+      # Release buffer on failure
+      context.release_buffer(buffer)
+      return context.err_at(
+        self,
+        source,
+        error_msgs[:minrep],
+        start_pos,
+        [break_on])
+    end
+
+    # Check consume_all requirement
+    if consume_all && source.chars_left > 0
+      # Release buffer on failure
+      context.release_buffer(buffer)
+      return context.err(
+        self,
+        source,
+        error_msgs[:unconsumed],
+        [break_on])
+    end
+
+    # Phase 2.2: Return lazy result instead of array
+    lazy_result = Parsanol::LazyResult.new(buffer, context)
+    return succ(lazy_result)
+  end
+
+  # General repetition parsing (extracted for clarity)
+  def try_repetition_general(source, context, consume_all)
+    occ = 0
+    start_pos = source.bytepos
+    
+    # Phase 2.2: Use buffer pool for lazy materialization
+    # Estimate size based on max, or default to 10
+    estimated_size = [max || 10, 10].min
+    buffer = context.acquire_buffer(size: estimated_size + 1)  # +1 for tag
+    buffer.push(@tag)  # Add tag first to maintain [:repetition, ...] structure
+
+    break_on = nil
+    loop do
+      success, value = parslet.apply(source, context, false)
+
+      break_on = value
+      break unless success
+
+      occ += 1
+      buffer.push(value)
+      
+      # If max is defined and reached, return lazy result
+      if max && occ >= max
+        lazy_result = Parsanol::LazyResult.new(buffer, context)
+        return succ(lazy_result)
+      end
+    end
+
+    # Last attempt to match parslet was a failure, failure reason in break_on.
+
+    # Greedy matcher has produced a failure. Check if occ (which will
+    # contain the number of successes) is >= min.
+    if occ < min
+      # Release buffer on failure
+      context.release_buffer(buffer)
+      return context.err_at(
+        self,
+        source,
+        error_msgs[:minrep],
+        start_pos,
+        [break_on])
+    end
+
+    # consume_all is true, that means that we're inside the part of the parser
+    # that should consume the input completely. Repetition failing here means
+    # probably that we didn't.
+    #
+    # We have a special clause to create an error here because otherwise
+    # break_on would get thrown away. It turns out, that contains very
+    # interesting information in a lot of cases.
+    #
+    if consume_all && source.chars_left > 0
+      # Release buffer on failure
+      context.release_buffer(buffer)
+      return context.err(
+        self,
+        source,
+        error_msgs[:unconsumed],
+        [break_on])
+    end
+
+    # Phase 2.2: Return lazy result instead of array
+    lazy_result = Parsanol::LazyResult.new(buffer, context)
+    return succ(lazy_result)
+  end
+
+  precedence REPETITION
+  def to_s_inner(prec)
+    minmax = "{#{min}, #{max}}"
+    minmax = '?' if min == 0 && max == 1
+
+    parslet.to_s(prec) + minmax
+  end
+
+  # FIRST set of repetition:
+  # - If min == 0 (can match empty), includes EPSILON
+  # - Always includes FIRST of the repeated parslet
+  def compute_first_set
+    result = parslet.first_set.dup
+    # If repetition can match zero times, add EPSILON
+    result.add(Parsanol::FirstSet::EPSILON) if min == 0
+    result
+  end
+end

data/lib/parsanol/atoms/scope.rb

@@ -0,0 +1,28 @@
+# Starts a new scope in the parsing process. Please also see the #captures
+# method.
+#
+class Parsanol::Atoms::Scope < Parsanol::Atoms::Base
+  attr_reader :block
+  def initialize(block)
+    super()
+
+    @block = block
+  end
+
+  def cached?
+    false
+  end
+
+  def apply(source, context, consume_all)
+    # Phase 55: Cache @block ivar to reduce lookup overhead
+    block = @block
+    context.scope do
+      parslet = block.call
+      return parslet.apply(source, context, consume_all)
+    end
+  end
+
+  def to_s_inner(prec)
+    "scope { #{block.call.to_s(prec)} }"
+  end
+end