parsanol 3.0.0

data/lib/parsanol/ast_visitor.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# Base class for AST visitors following the Visitor pattern
+# This separates tree traversal logic from transformation logic
+# making the code more maintainable and extensible.
+module Parsanol
+  # Base visitor class that traverses the Parslet AST
+  # Subclasses override visit_* methods to perform transformations
+  class ASTVisitor
+    # Visit a parslet and its children
+    # Subclasses should override specific visit_* methods
+    # @param parslet [Parsanol::Atoms::Base] parslet to visit
+    # @return [Parsanol::Atoms::Base] transformed parslet
+    def visit(parslet)
+      case parslet
+      when Parsanol::Atoms::Sequence
+        visit_sequence(parslet)
+      when Parsanol::Atoms::Alternative
+        visit_alternative(parslet)
+      when Parsanol::Atoms::Repetition
+        visit_repetition(parslet)
+      when Parsanol::Atoms::Lookahead
+        visit_lookahead(parslet)
+      when Parsanol::Atoms::Named
+        visit_named(parslet)
+      when Parsanol::Atoms::Str
+        visit_str(parslet)
+      when Parsanol::Atoms::Re
+        visit_re(parslet)
+      else
+        # Leaf nodes or unknown types - return as-is
+        parslet
+      end
+    end
+
+    # Visit a sequence node
+    # Default implementation visits children and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Sequence] sequence to visit
+    # @return [Parsanol::Atoms::Base] transformed sequence
+    def visit_sequence(parslet)
+      new_parslets = parslet.parslets.map { |p| visit(p) }
+      if new_parslets == parslet.parslets
+        parslet
+      else
+        Parsanol::Atoms::Sequence.new(*new_parslets)
+      end
+    end
+
+    # Visit an alternative node
+    # Default implementation visits children and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Alternative] alternative to visit
+    # @return [Parsanol::Atoms::Base] transformed alternative
+    def visit_alternative(parslet)
+      new_alternatives = parslet.alternatives.map { |p| visit(p) }
+      if new_alternatives == parslet.alternatives
+        parslet
+      else
+        Parsanol::Atoms::Alternative.new(*new_alternatives)
+      end
+    end
+
+    # Visit a repetition node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Repetition] repetition to visit
+    # @return [Parsanol::Atoms::Base] transformed repetition
+    def visit_repetition(parslet)
+      new_parslet = visit(parslet.parslet)
+      if new_parslet.equal?(parslet.parslet)
+        parslet
+      else
+        Parsanol::Atoms::Repetition.new(
+          new_parslet,
+          parslet.min,
+          parslet.max,
+          parslet.instance_variable_get(:@tag)
+        )
+      end
+    end
+
+    # Visit a lookahead node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Lookahead] lookahead to visit
+    # @return [Parsanol::Atoms::Base] transformed lookahead
+    def visit_lookahead(parslet)
+      new_bound = visit(parslet.bound_parslet)
+      if new_bound.equal?(parslet.bound_parslet)
+        parslet
+      else
+        Parsanol::Atoms::Lookahead.new(new_bound, parslet.positive)
+      end
+    end
+
+    # Visit a named node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Named] named to visit
+    # @return [Parsanol::Atoms::Base] transformed named
+    def visit_named(parslet)
+      new_parslet = visit(parslet.parslet)
+      if new_parslet.equal?(parslet.parslet)
+        parslet
+      else
+        Parsanol::Atoms::Named.new(new_parslet, parslet.name)
+      end
+    end
+
+    # Visit a string literal node
+    # Default implementation returns as-is (leaf node)
+    # @param parslet [Parsanol::Atoms::Str] string to visit
+    # @return [Parsanol::Atoms::Base] transformed string
+    def visit_str(parslet)
+      parslet
+    end
+
+    # Visit a regex node
+    # Default implementation returns as-is (leaf node)
+    # @param parslet [Parsanol::Atoms::Re] regex to visit
+    # @return [Parsanol::Atoms::Base] transformed regex
+    def visit_re(parslet)
+      parslet
+    end
+  end
+end

data/lib/parsanol/atoms/alternative.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Alternative during matching. Contains a list of parslets that is tried each
+# one in turn. Only fails if all alternatives fail.
+#
+# Example:
+#
+#   str('a') | str('b')   # matches either 'a' or 'b'
+#
+class Parsanol::Atoms::Alternative < Parsanol::Atoms::Base
+  attr_reader :alternatives
+
+  # Constructs an Alternative instance using all given parslets in the order
+  # given. This is what happens if you call '|' on existing parslets, like
+  # this:
+  #
+  #   str('a') | str('b')
+  #
+  def initialize(*alternatives)
+    super()
+
+    @alternatives = alternatives
+
+    # Phase 60: Pre-compute and freeze error message
+    @error_msg = "Expected one of #{alternatives.inspect}".freeze
+  end
+
+  #---
+  # Don't construct a hanging tree of Alternative parslets, instead store them
+  # all here. This reduces the number of objects created.
+  #+++
+  def |(parslet)
+    # Phase 25: Alternative Flattening (similar to Phase 21 for Sequence)
+    # Flatten nested alternatives: (A | B) | C becomes Alternative(A, B, C)
+    # instead of Alternative(Alternative(A, B), C)
+    new_alts = if parslet.is_a?(Parsanol::Atoms::Alternative)
+      @alternatives + parslet.alternatives
+    else
+      @alternatives + [parslet]
+    end
+    self.class.new(*new_alts)
+  end
+
+
+  def try(source, context, consume_all)
+    # Phase 52: Cache @alternatives ivar to reduce lookup overhead
+    alternatives = @alternatives
+
+    # Fast paths for common alternative sizes (avoid iteration overhead)
+    case alternatives.size
+    when 2
+      success, value = alternatives[0].apply(source, context, consume_all)
+      return [success, value] if success
+      success2, value2 = alternatives[1].apply(source, context, consume_all)
+      return [success2, value2] if success2
+      return context.err(self, source, @error_msg, [value, value2])
+    when 3
+      success, value = alternatives[0].apply(source, context, consume_all)
+      return [success, value] if success
+      success2, value2 = alternatives[1].apply(source, context, consume_all)
+      return [success2, value2] if success2
+      success3, value3 = alternatives[2].apply(source, context, consume_all)
+      return [success3, value3] if success3
+      return context.err(self, source, @error_msg, [value, value2, value3])
+    end
+
+    # General case: Optimize by not allocating error array until we know all alternatives fail
+    # This saves significant allocation overhead when early alternatives succeed
+    errors = nil
+
+    alternatives.each do |a|
+      success, value = result = a.apply(source, context, consume_all)
+      return result if success
+
+      # Lazily allocate errors array only if needed
+      errors ||= []
+      errors << value
+    end
+
+    # If we reach this point, all alternatives have failed.
+    context.err(self, source, @error_msg, errors)
+  end
+
+  precedence ALTERNATE
+  def to_s_inner(prec)
+    alternatives.map { |a| a.to_s(prec) }.join(' / ')
+  end
+
+  # FIRST set of alternative is union of all alternatives' FIRST sets
+  # This is the key computation for cut operator insertion:
+  # If FIRST(alt1) ∩ FIRST(alt2) = ∅, we can insert a cut after alt1
+  def compute_first_set
+    return Set.new if alternatives.empty?
+
+    alternatives.map(&:first_set).reduce(&:union)
+  end
+end

data/lib/parsanol/atoms/base.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+# Base class for all parslets, handles orchestration of calls and implements
+# a lot of the operator and chaining methods.
+#
+# Also see Parsanol::Atoms::DSL chaining parslet atoms together.
+#
+class Parsanol::Atoms::Base
+  include Parsanol::Atoms::Precedence
+  include Parsanol::Atoms::DSL
+  include Parsanol::Atoms::CanFlatten
+  include Parsanol::FirstSet
+
+  # Parslet label as provided in grammar
+  attr_accessor :label
+
+  # Phase 61: Frozen error message for unknown input
+  ERROR_UNKNOWN_INPUT = "Don't know what to do with ".freeze
+
+  # Given a string or an IO object, this will attempt a parse of its contents
+  # and return a result. If the parse fails, a Parsanol::ParseFailed exception
+  # will be thrown.
+  #
+  # @param io [String, Source] input for the parse process
+  # @option options [Parsanol::ErrorReporter] :reporter error reporter to use,
+  #   defaults to Parsanol::ErrorReporter::Tree
+  # @option options [Boolean] :prefix Should a prefix match be accepted?
+  #   (default: false)
+  # @return [Hash, Array, Parsanol::Slice] PORO (Plain old Ruby object) result
+  #   tree
+  #
+  def parse(io, options={})
+    source = io.respond_to?(:line_and_column) ?
+      io :
+      Parsanol::Source.new(io)
+
+    # Try to cheat. Assuming that we'll be able to parse the input, don't
+    # run error reporting code.
+    success, value = setup_and_apply(source, nil, !options[:prefix])
+
+    # If we didn't succeed the parse, raise an exception for the user.
+    # Stack trace will be off, but the error tree should explain the reason
+    # it failed.
+    unless success
+      # Cheating has not paid off. Now pay the cost: Rerun the parse,
+      # gathering error information in the process.
+      reporter = options[:reporter] || Parsanol::ErrorReporter::Tree.new
+      source.bytepos = 0
+      success, value = setup_and_apply(source, reporter, !options[:prefix])
+
+      fail "Assertion failed: success was true when parsing with reporter" \
+        if success
+
+      # Value is a Parsanol::Cause, which can be turned into an exception:
+      value.raise
+
+      fail "NEVER REACHED"
+    end
+
+    # assert: success is true
+
+    # Extra input is now handled inline with the rest of the parsing. If
+    # really we have success == true, prefix: false and still some input
+    # is left dangling, that is a BUG.
+    if !options[:prefix] && source.chars_left > 0
+      fail "BUG: New error strategy should not reach this point."
+    end
+
+    return flatten(value)
+  end
+
+  # Creates a context for parsing and applies the current atom to the input.
+  # Returns the parse result.
+  #
+  # @return [<Boolean, Object>] Result of the parse. If the first member is
+  #   true, the parse has succeeded.
+  def setup_and_apply(source, error_reporter, consume_all)
+    # Session 13: Pass parser class for per-parser cache threshold selection
+    # If self is a Parser instance, pass its class for threshold lookup
+    parser_class = self.is_a?(Parsanol::Parser) ? self.class : nil
+    context = Parsanol::Atoms::Context.new(error_reporter, parser_class: parser_class)
+    apply(source, context, consume_all)
+  end
+
+  # Calls the #try method of this parslet. Success consumes input, error will
+  # rewind the input.
+  #
+  # @param source [Parsanol::Source] source to read input from
+  # @param context [Parsanol::Atoms::Context] context to use for the parsing
+  # @param consume_all [Boolean] true if the current parse must consume
+  #   all input by itself.
+  def apply(source, context, consume_all=false)
+    old_pos = source.bytepos
+
+    success, _ = result = context.try_with_cache(self, source, consume_all)
+
+    if success
+      # Notify context
+      context.succ(source)
+      # If a consume_all parse was made and doesn't result in the consumption
+      # of all the input, that is considered an error.
+      if consume_all && source.chars_left>0
+        # Read 10 characters ahead. Why ten? I don't know.
+        offending_pos   = source.bytepos
+        offending_input = source.consume(10)
+
+        # Rewind input (as happens always in error case)
+        source.bytepos  = old_pos
+
+        return context.err_at(
+          self,
+          source,
+          ERROR_UNKNOWN_INPUT + offending_input.to_s.inspect,
+          offending_pos
+        )
+      end
+
+      # Looks like the parse was successful after all. Don't rewind the input.
+      return result
+    end
+
+    # We only reach this point if the parse has failed. Rewind the input.
+    source.bytepos = old_pos
+    return result
+  end
+
+  # Override this in your Atoms::Base subclasses to implement parsing
+  # behaviour.
+  #
+  def try(source, context, consume_all)
+    raise NotImplementedError, \
+      "Atoms::Base doesn't have behaviour, please implement #try(source, context)."
+  end
+
+  # Returns true if this atom can be cached in the packrat cache. Most parslet
+  # atoms are cached, so this always returns true, unless overridden.
+  #
+  def cached?
+    true
+  end
+
+  # Returns true if this atom produces flat results by construction
+  # (no nested arrays/hashes that need flattening). This allows flatten
+  # to skip processing for atoms that are known to produce simple values.
+  #
+  # Session 13: Flatten optimization to reduce 5.27% overhead
+  # Atoms like Str and Re always produce strings (Parsanol::Slice),
+  # which don't need flattening.
+  #
+  # @return [Boolean] true if results are flat by construction
+  def flat?
+    false  # Default: assume needs flattening
+  end
+
+  # Debug printing - in Treetop syntax.
+  #
+  def self.precedence(prec)
+    define_method(:precedence) { prec }
+  end
+  precedence BASE
+  def to_s(outer_prec=OUTER)
+    str = label || to_s_inner(precedence)
+    if outer_prec < precedence
+      "(#{str})"
+    else
+      str
+    end
+  end
+  def inspect
+    to_s(OUTER)
+  end
+private
+
+  # Produces an instance of Success and returns it.
+  #
+  # Phase 57a-57b: Frozen constants for common result patterns to reduce allocations.
+  # These constants are used extensively in hot paths to avoid creating new arrays.
+  #
+  SUCCESS_NIL = [true, nil].freeze
+
+  # Common patterns for repetition results with empty values
+  # Format: [true, [tag]] where tag is the repetition marker
+  # Pre-allocated for common tags to avoid array creation
+  EMPTY_ARRAY = [].freeze
+
+  # Phase 57b: Additional frozen constants for tagged empty arrays
+  # These are common in repetitions that match 0 times (.maybe, .repeat(0,n))
+  EMPTY_REPETITION_ARRAY = [:repetition].freeze
+  SUCCESS_EMPTY_REPETITION = [true, EMPTY_REPETITION_ARRAY].freeze
+
+  EMPTY_SEQUENCE_ARRAY = [:sequence].freeze
+  SUCCESS_EMPTY_SEQUENCE = [true, EMPTY_SEQUENCE_ARRAY].freeze
+
+  # Phase 57c: Additional frozen constants for common patterns
+  EMPTY_HASH = {}.freeze
+  SUCCESS_EMPTY_HASH = [true, EMPTY_HASH].freeze
+
+  # Common single-element arrays for captures and tags
+  EMPTY_CAPTURE_ARRAY = [:capture].freeze
+  SUCCESS_EMPTY_CAPTURE = [true, EMPTY_CAPTURE_ARRAY].freeze
+
+  def succ(result)
+    return SUCCESS_NIL if result.nil?
+    # Check for empty array (common in repetitions with 0 matches)
+    return [true, EMPTY_ARRAY] if result.equal?(EMPTY_ARRAY)
+    # Check for empty hash (common in named captures with no matches)
+    return SUCCESS_EMPTY_HASH if result.equal?(EMPTY_HASH)
+    # Check for common tagged empty arrays
+    return SUCCESS_EMPTY_REPETITION if result.equal?(EMPTY_REPETITION_ARRAY)
+    return SUCCESS_EMPTY_SEQUENCE if result.equal?(EMPTY_SEQUENCE_ARRAY)
+    return SUCCESS_EMPTY_CAPTURE if result.equal?(EMPTY_CAPTURE_ARRAY)
+    [true, result]
+  end
+end

data/lib/parsanol/atoms/can_flatten.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+
+module Parsanol::Atoms
+  # A series of helper functions that have the common topic of flattening
+  # result values into the intermediary tree that consists of Ruby Hashes and
+  # Arrays.
+  #
+  # This module has one main function, #flatten, that takes an annotated
+  # structure as input and returns the reduced form that users expect from
+  # Atom#parse.
+  #
+  # NOTE: Since all of these functions are just that, functions without
+  # side effects, they are in a module and not in a class. Its hard to draw
+  # the line sometimes, but this is beyond.
+  #
+  module CanFlatten
+    # Takes a mixed value coming out of a parslet and converts it to a return
+    # value for the user by dropping things and merging hashes.
+    #
+    # Named is set to true if this result will be embedded in a Hash result from
+    # naming something using <code>.as(...)</code>. It changes the folding
+    # semantics of repetition.
+    #
+    def flatten(value, named=false)
+      # Passes through everything that isn't an array of things
+      # Phase 43: Use simpler check - if it's not an Array, return as-is
+      return value unless value.is_a?(Array)
+
+      # Extracts the s-expression tag
+      tag = value[0]
+
+      # Phase 43: Optimize flattening - reduce method call overhead
+      # For single element arrays (common case), handle directly
+      tail_size = value.size - 1
+      if tail_size == 1
+        flattened = flatten(value[1])
+        case tag
+          when :sequence
+            return flattened
+          when :maybe
+            return named ? flattened : (flattened || '')
+          when :repetition
+            return flatten_repetition([flattened], named)
+        end
+      end
+
+      # Flatten each element
+      result = Array.new(tail_size)
+      i = 0
+      while i < tail_size
+        result[i] = flatten(value[i + 1])
+        i += 1
+      end
+
+      case tag
+        when :sequence
+          return flatten_sequence(result)
+        when :maybe
+          return named ? result.first : result.first || ''
+        when :repetition
+          return flatten_repetition(result, named)
+      end
+
+      fail "BUG: Unknown tag #{tag.inspect}."
+    end
+
+    # Lisp style fold left where the first element builds the basis for
+    # an inject. Optimized with early return and reduced method calls.
+    #
+    def foldl(list, &block)
+      len = list.size
+      return '' if len == 0
+      return list[0] if len == 1  # Fast path for single element
+
+      result = list[0]
+      i = 1
+      while i < len
+        result = block.call(result, list[i])
+        i += 1
+      end
+      result
+    end
+
+    # Flatten results from a sequence of parslets.
+    #
+    # @api private
+    #
+    def flatten_sequence(list)
+      foldl(list.compact) { |r, e|        # and then merge flat elements
+        merge_fold(r, e)
+      }
+    end
+    # @api private
+    # Phase 43: Optimized merge_fold - reduce repeated class checks
+    def merge_fold(l, r)
+      l_class = l.class
+      r_class = r.class
+
+      # equal pairs: merge. ----------------------------------------------------
+      if l_class == r_class
+        if l_class == Hash
+          warn_about_duplicate_keys(l, r)
+          return l.merge(r)
+        else
+          return l + r
+        end
+      end
+
+      # Phase 43: Cache instance_of? checks to avoid repeated method calls
+      # unequal pairs: hoist to same level. ------------------------------------
+      l_is_slice = l.instance_of?(Parsanol::Slice)
+      r_is_slice = r.instance_of?(Parsanol::Slice)
+      l_is_str = l_class == String || l_is_slice
+      r_is_str = r_class == String || r_is_slice
+
+      # Maybe classes are not equal, but both are stringlike?
+      if l_is_str && r_is_str
+        # if we're merging a String with a Slice, the slice wins.
+        return r if r_is_slice
+        return l if l_is_slice
+
+        fail "NOTREACHED: What other stringlike classes are there?"
+      end
+
+      # special case: If one of them is a string/slice, the other is more important
+      return l if r_is_str
+      return r if l_is_str
+
+      # otherwise just create an array for one of them to live in
+      return l + [r] if r_class == Hash
+      return [l] + r if l_class == Hash
+
+      fail "Unhandled case when foldr'ing sequence."
+    end
+
+    # Flatten results from a repetition of a single parslet. named indicates
+    # whether the user has named the result or not. If the user has named
+    # the results, we want to leave an empty list alone - otherwise it is
+    # turned into an empty string.
+    #
+    # @api private
+    #
+    # Phase 43: Optimized flatten_repetition - reduce array iterations
+    def flatten_repetition(list, named)
+      # Phase 43: Single pass to check for hashes and arrays
+      has_hash = false
+      has_array = false
+
+      i = 0
+      len = list.size
+      while i < len
+        e = list[i]
+        has_hash = true if e.instance_of?(Hash)
+        has_array = true if e.instance_of?(Array)
+        break if has_hash && has_array  # Early exit if both found
+        i += 1
+      end
+
+      if has_hash
+        # If keyed subtrees are in the array, we'll want to discard all
+        # strings inbetween. To keep them, name them.
+        return list.select { |e| e.instance_of?(Hash) }
+      end
+
+      if has_array
+        # If any arrays are nested in this array, flatten all arrays to this
+        # level.
+        return list.
+          select { |e| e.instance_of?(Array) }.
+          flatten(1)
+      end
+
+      # Consistent handling of empty lists, when we act on a named result
+      return [] if named && list.empty?
+
+      # If there are only strings, concatenate them and return that.
+      foldl(list.compact) { |s,e| s+e }
+    end
+
+    # That annoying warning 'Duplicate subtrees while merging result' comes
+    # from here. You should add more '.as(...)' names to your intermediary tree.
+    #
+    def warn_about_duplicate_keys(h1, h2)
+      d = h1.keys & h2.keys
+      unless d.empty?
+        warn "Duplicate subtrees while merging result of \n  #{self.inspect}\nonly the values"+
+             " of the latter will be kept. (keys: #{d.inspect})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/capture.rb

@@ -0,0 +1,41 @@
+
+# Stores the result of matching an atom against input in the #captures in
+# parse context. Doing so will allow you to pull parts of the ongoing parse
+# out later and use them to match other pieces of input.
+#
+# Example:
+#   # After this, context.captures[:an_a] returns 'a'
+#   str('a').capture(:an_a)
+#
+#   # Capture and use of the capture: (matches either 'aa' or 'bb')
+#   match['ab'].capture(:first) >>
+#     dynamic { |src, ctx| str(ctx.captures[:first]) }
+#
+class Parsanol::Atoms::Capture < Parsanol::Atoms::Base
+  attr_reader :parslet, :name
+
+  def initialize(parslet, name)
+    super()
+
+    @parslet, @name = parslet, name
+  end
+
+  def apply(source, context, consume_all)
+    # Phase 55: Cache ivars to reduce lookup overhead
+    parslet = @parslet
+    name = @name
+
+    success, value = result = parslet.apply(source, context, consume_all)
+
+    if success
+      context.captures[name.to_sym] =
+        flatten(value)
+    end
+
+    return result
+  end
+
+  def to_s_inner(prec)
+    "(#{name.inspect} = #{parslet.to_s(prec)})"
+  end
+end