parsanol 1.0.0

data/lib/parsanol/ast_visitor.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# Base class for AST visitors following the Visitor pattern
+# This separates tree traversal logic from transformation logic
+# making the code more maintainable and extensible.
+module Parsanol
+  # Base visitor class that traverses the Parslet AST
+  # Subclasses override visit_* methods to perform transformations
+  class ASTVisitor
+    # Visit a parslet and its children
+    # Subclasses should override specific visit_* methods
+    # @param parslet [Parsanol::Atoms::Base] parslet to visit
+    # @return [Parsanol::Atoms::Base] transformed parslet
+    def visit(parslet)
+      case parslet
+      when Parsanol::Atoms::Sequence
+        visit_sequence(parslet)
+      when Parsanol::Atoms::Alternative
+        visit_alternative(parslet)
+      when Parsanol::Atoms::Repetition
+        visit_repetition(parslet)
+      when Parsanol::Atoms::Lookahead
+        visit_lookahead(parslet)
+      when Parsanol::Atoms::Named
+        visit_named(parslet)
+      when Parsanol::Atoms::Str
+        visit_str(parslet)
+      when Parsanol::Atoms::Re
+        visit_re(parslet)
+      else
+        # Leaf nodes or unknown types - return as-is
+        parslet
+      end
+    end
+
+    # Visit a sequence node
+    # Default implementation visits children and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Sequence] sequence to visit
+    # @return [Parsanol::Atoms::Base] transformed sequence
+    def visit_sequence(parslet)
+      new_parslets = parslet.parslets.map { |p| visit(p) }
+      if new_parslets == parslet.parslets
+        parslet
+      else
+        Parsanol::Atoms::Sequence.new(*new_parslets)
+      end
+    end
+
+    # Visit an alternative node
+    # Default implementation visits children and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Alternative] alternative to visit
+    # @return [Parsanol::Atoms::Base] transformed alternative
+    def visit_alternative(parslet)
+      new_alternatives = parslet.alternatives.map { |p| visit(p) }
+      if new_alternatives == parslet.alternatives
+        parslet
+      else
+        Parsanol::Atoms::Alternative.new(*new_alternatives)
+      end
+    end
+
+    # Visit a repetition node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Repetition] repetition to visit
+    # @return [Parsanol::Atoms::Base] transformed repetition
+    def visit_repetition(parslet)
+      new_parslet = visit(parslet.parslet)
+      if new_parslet.equal?(parslet.parslet)
+        parslet
+      else
+        Parsanol::Atoms::Repetition.new(
+          new_parslet,
+          parslet.min,
+          parslet.max,
+          parslet.instance_variable_get(:@tag)
+        )
+      end
+    end
+
+    # Visit a lookahead node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Lookahead] lookahead to visit
+    # @return [Parsanol::Atoms::Base] transformed lookahead
+    def visit_lookahead(parslet)
+      new_bound = visit(parslet.bound_parslet)
+      if new_bound.equal?(parslet.bound_parslet)
+        parslet
+      else
+        Parsanol::Atoms::Lookahead.new(new_bound, parslet.positive)
+      end
+    end
+
+    # Visit a named node
+    # Default implementation visits child and reconstructs if changed
+    # @param parslet [Parsanol::Atoms::Named] named to visit
+    # @return [Parsanol::Atoms::Base] transformed named
+    def visit_named(parslet)
+      new_parslet = visit(parslet.parslet)
+      if new_parslet.equal?(parslet.parslet)
+        parslet
+      else
+        Parsanol::Atoms::Named.new(new_parslet, parslet.name)
+      end
+    end
+
+    # Visit a string literal node
+    # Default implementation returns as-is (leaf node)
+    # @param parslet [Parsanol::Atoms::Str] string to visit
+    # @return [Parsanol::Atoms::Base] transformed string
+    def visit_str(parslet)
+      parslet
+    end
+
+    # Visit a regex node
+    # Default implementation returns as-is (leaf node)
+    # @param parslet [Parsanol::Atoms::Re] regex to visit
+    # @return [Parsanol::Atoms::Base] transformed regex
+    def visit_re(parslet)
+      parslet
+    end
+  end
+end

data/lib/parsanol/atoms/alternative.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+# Ordered choice - tries alternatives left-to-right, returning first success.
+# Fails only if all alternatives fail.
+#
+# @example Simple choice
+#   str('a') | str('b')  # matches 'a' or 'b'
+#
+# This is PEG ordered choice - no backtracking to later alternatives.
+#
+module Parsanol
+  module Atoms
+    class Alternative < Parsanol::Atoms::Base
+      # @return [Array<Parsanol::Atoms::Base>] alternative parsers
+      attr_reader :alternatives
+
+      # Creates a new choice.
+      #
+      # @param options [Array<Parsanol::Atoms::Base>] alternatives
+      def initialize(*options)
+        super()
+        @alternatives = options
+        @choice_error = "Expected one of #{options.inspect}"
+      end
+
+      # Adds an alternative with flattening.
+      #
+      # @param parser [Parsanol::Atoms::Base] new alternative
+      # @return [Parsanol::Atoms::Alternative] flattened choice
+      def |(other)
+        expanded = if other.is_a?(Parsanol::Atoms::Alternative)
+                     @alternatives + other.alternatives
+                   else
+                     @alternatives + [other]
+                   end
+        self.class.new(*expanded)
+      end
+
+      # Tries each alternative in order.
+      #
+      # @param source [Parsanol::Source] input
+      # @param context [Parsanol::Atoms::Context] context
+      # @param consume_all [Boolean] require full consumption
+      # @return [Array(Boolean, Object)] result
+      def try(source, context, consume_all)
+        options = @alternatives
+        count = options.size
+
+        # Optimized paths for common sizes
+        case count
+        when 2
+          try_two(options[0], options[1], source, context, consume_all)
+        when 3
+          try_three(options[0], options[1], options[2], source, context, consume_all)
+        else
+          try_many(options, source, context, consume_all)
+        end
+      end
+
+      precedence CHOICE
+
+      # String representation.
+      #
+      # @param prec [Integer] precedence
+      # @return [String]
+      def to_s_inner(prec)
+        @alternatives.map { |a| a.to_s(prec) }.join(' / ')
+      end
+
+      # FIRST set is union of all alternatives' FIRST sets.
+      #
+      # @return [Set]
+      def compute_first_set
+        return Set.new if @alternatives.empty?
+
+        @alternatives.map(&:first_set).reduce(&:union)
+      end
+
+      private
+
+      # Two-alternative fast path
+      def try_two(a1, a2, source, context, consume_all)
+        success, value1 = a1.apply(source, context, consume_all)
+        return [success, value1] if success
+
+        success, value2 = a2.apply(source, context, consume_all)
+        return [success, value2] if success
+
+        context.err(self, source, @choice_error, [value1, value2])
+      end
+
+      # Three-alternative fast path
+      def try_three(a1, a2, a3, source, context, consume_all)
+        success, value1 = a1.apply(source, context, consume_all)
+        return [success, value1] if success
+
+        success, value2 = a2.apply(source, context, consume_all)
+        return [success, value2] if success
+
+        success, value3 = a3.apply(source, context, consume_all)
+        return [success, value3] if success
+
+        context.err(self, source, @choice_error, [value1, value2, value3])
+      end
+
+      # General case for N alternatives
+      def try_many(options, source, context, consume_all)
+        errors = nil
+
+        options.each do |alt|
+          success, value = alt.apply(source, context, consume_all)
+          return [success, value] if success
+
+          errors ||= []
+          errors << value
+        end
+
+        context.err(self, source, @choice_error, errors)
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/base.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+# Base class for all parser atoms. Handles parsing orchestration,
+# memoization, error handling, and result processing.
+#
+# Concrete atoms must implement #try(source, context, consume_all).
+#
+# @abstract Implement #try to create custom parser atoms
+module Parsanol
+  module Atoms
+    class Base
+      include Parsanol::Atoms::Precedence
+      include Parsanol::Atoms::DSL
+      include Parsanol::Atoms::CanFlatten
+      include Parsanol::FirstSet
+
+      # Label used for error messages (optional)
+      attr_accessor :label
+
+      # Error message for unconsumed input
+      UNCONSUMED_INPUT_MSG = "Don't know what to do with "
+
+      # Primary parsing interface. Takes a string or Source and returns
+      # the parsed tree, or raises ParseFailed on error.
+      #
+      # @param source [String, Parsanol::Source] input to parse
+      # @param options [Hash] parsing options
+      # @option options [Parsanol::ErrorReporter] :reporter error collector
+      # @option options [Boolean] :prefix allow partial parse (default: false)
+      # @return [Object] the parsed result
+      # @raise [Parsanol::ParseFailed] on parse failure
+      def parse(source, options = {})
+        input = normalize_input(source)
+        must_consume_all = !options[:prefix]
+
+        # Initial parse attempt (no error collection)
+        success, value = run_with_context(input, nil, must_consume_all)
+        return finalize_result(value) if success
+
+        # Reparse with error reporting for diagnostics
+        report_detailed_error(input, must_consume_all, options[:reporter], value)
+      end
+
+      # Creates a new parsing context and executes the atom.
+      #
+      # @param input [Parsanol::Source] the source
+      # @param reporter [Object, nil] error reporter
+      # @param consume_all [Boolean] require complete consumption
+      # @return [Array(Boolean, Object)] outcome tuple
+      def run_with_context(input, reporter, consume_all)
+        parser_class = detect_parser_class
+        context = Parsanol::Atoms::Context.new(reporter, parser_class: parser_class)
+        apply(input, context, consume_all)
+      end
+
+      # Core execution method. Manages position, caching, and error handling.
+      #
+      # @param input [Parsanol::Source] source to parse
+      # @param context [Parsanol::Atoms::Context] parsing state
+      # @param consume_all [Boolean] consume entire input
+      # @return [Array(Boolean, Object)] outcome pair
+      def apply(input, context, consume_all = false)
+        position_before = input.bytepos
+        outcome = context.try_with_cache(self, input, consume_all)
+        succeeded = outcome.first
+
+        return handle_failure(input, position_before, outcome) unless succeeded
+
+        context.succ(input)
+
+        # Verify full consumption when required
+        return unconsumed_error(input, context, position_before) if consume_all && input.chars_left.positive?
+
+        outcome
+      end
+
+      # Abstract matching method - override in subclasses.
+      #
+      # @param input [Parsanol::Source] source
+      # @param context [Parsanol::Atoms::Context] context
+      # @param consume_all [Boolean] consume all flag
+      # @return [Array(Boolean, Object)] parse result
+      # @raise [NotImplementedError] if not overridden
+      def try(input, context, consume_all)
+        raise NotImplementedError,
+              'Atom must implement #try(source, context, consume_all)'
+      end
+
+      # Whether packrat caching benefits this atom.
+      # Override to disable caching for simple atoms.
+      #
+      # @return [Boolean]
+      def cached?
+        true
+      end
+
+      # Whether this atom produces flat results.
+      # When true, flattening can be skipped.
+      #
+      # @return [Boolean]
+      def flat?
+        false
+      end
+
+      # DSL for setting precedence level (for pretty-printing).
+      #
+      # @param level [Integer] precedence value
+      def self.precedence(level)
+        define_method(:precedence) { level }
+      end
+      precedence ATOM
+
+      # String representation with precedence-aware parenthesization.
+      #
+      # @param outer [Integer] caller's precedence
+      # @return [String]
+      def to_s(outer = TOP)
+        text = label || to_s_inner(precedence)
+        outer < precedence ? "(#{text})" : text
+      end
+
+      def inspect
+        to_s(TOP)
+      end
+
+      protected
+
+      # Pre-allocated constant result tuples
+      NIL_OK = [true, nil].freeze
+      EMPTY_ARR = [].freeze
+      REP_TAG = [:repetition].freeze
+      REP_OK = [true, REP_TAG].freeze
+      SEQ_TAG = [:sequence].freeze
+      SEQ_OK = [true, SEQ_TAG].freeze
+      EMPTY_MAP = {}.freeze
+      MAP_OK = [true, EMPTY_MAP].freeze
+      CAP_TAG = [:capture].freeze
+      CAP_OK = [true, CAP_TAG].freeze
+
+      # Creates a success tuple.
+      #
+      # @param data [Object] the value
+      # @return [Array(true, Object)]
+      def ok(data)
+        return NIL_OK if data.nil?
+        return [true, EMPTY_ARR] if data.equal?(EMPTY_ARR)
+        return MAP_OK if data.equal?(EMPTY_MAP)
+        return REP_OK if data.equal?(REP_TAG)
+        return SEQ_OK if data.equal?(SEQ_TAG)
+        return CAP_OK if data.equal?(CAP_TAG)
+
+        [true, data]
+      end
+
+      # Alias for ok (legacy compatibility)
+      alias succ ok
+
+      private
+
+      # Converts raw input to Source if needed.
+      def normalize_input(source)
+        source.respond_to?(:line_and_column) ? source : Parsanol::Source.new(source)
+      end
+
+      # Detects if we're in a Parser context.
+      def detect_parser_class
+        is_a?(Parsanol::Parser) ? self.class : nil
+      end
+
+      # Handles parse failure by restoring position.
+      def handle_failure(input, saved_pos, outcome)
+        input.bytepos = saved_pos
+        outcome
+      end
+
+      # Creates error for unconsumed input.
+      def unconsumed_error(input, context, saved_pos)
+        excess_pos = input.bytepos
+        preview = input.consume(10)
+        input.bytepos = saved_pos
+        context.err_at(self, input, UNCONSUMED_INPUT_MSG + preview.to_s.inspect, excess_pos)
+      end
+
+      # Reports detailed error by reparsing with reporter.
+      def report_detailed_error(input, consume_all, reporter, _initial_error)
+        input.bytepos = 0
+        error_reporter = reporter || Parsanol::ErrorReporter::Tree.new
+        success, cause = run_with_context(input, error_reporter, consume_all)
+
+        # Second parse should also fail
+        raise 'Invariant violation: parse succeeded during error reporting' if success
+
+        cause.raise
+      end
+
+      # Finalizes result by flattening.
+      def finalize_result(value)
+        flatten(value)
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/can_flatten.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module Atoms
+    # A series of helper functions that have the common topic of flattening
+    # result values into the intermediary tree that consists of Ruby Hashes and
+    # Arrays.
+    #
+    # This module has one main function, #flatten, that takes an annotated
+    # structure as input and returns the reduced form that users expect from
+    # Atom#parse.
+    #
+    # NOTE: Since all of these functions are just that, functions without
+    # side effects, they are in a module and not in a class. Its hard to draw
+    # the line sometimes, but this is beyond.
+    #
+    module CanFlatten
+      # Takes a mixed value coming out of a parslet and converts it to a return
+      # value for the user by dropping things and merging hashes.
+      #
+      # Named is set to true if this result will be embedded in a Hash result from
+      # naming something using <code>.as(...)</code>. It changes the folding
+      # semantics of repetition.
+      #
+      def flatten(value, named = false)
+        # Passes through everything that isn't an array of things
+        # Phase 43: Use simpler check - if it's not an Array, return as-is
+        return value unless value.is_a?(Array)
+
+        # Extracts the s-expression tag
+        tag = value[0]
+
+        # Phase 43: Optimize flattening - reduce method call overhead
+        # For single element arrays (common case), handle directly
+        tail_size = value.size - 1
+        if tail_size == 1
+          flattened = flatten(value[1])
+          case tag
+          when :sequence
+            return flattened
+          when :maybe
+            return named ? flattened : (flattened || '')
+          when :repetition
+            return flatten_repetition([flattened], named)
+          end
+        end
+
+        # Flatten each element
+        result = Array.new(tail_size)
+        i = 0
+        while i < tail_size
+          result[i] = flatten(value[i + 1])
+          i += 1
+        end
+
+        case tag
+        when :sequence
+          return flatten_sequence(result)
+        when :maybe
+          return named ? result.first : result.first || ''
+        when :repetition
+          return flatten_repetition(result, named)
+        end
+
+        raise "BUG: Unknown tag #{tag.inspect}."
+      end
+
+      # Lisp style fold left where the first element builds the basis for
+      # an inject. Optimized with early return and reduced method calls.
+      #
+      def foldl(list, &block)
+        len = list.size
+        return '' if len.zero?
+        return list[0] if len == 1 # Fast path for single element
+
+        result = list[0]
+        i = 1
+        while i < len
+          result = block.call(result, list[i])
+          i += 1
+        end
+        result
+      end
+
+      # Flatten results from a sequence of parslets.
+      #
+      # @api private
+      #
+      def flatten_sequence(list)
+        foldl(list.compact) do |r, e| # and then merge flat elements
+          merge_fold(r, e)
+        end
+      end
+
+      # @api private
+      # Phase 43: Optimized merge_fold - reduce repeated class checks
+      def merge_fold(l, r)
+        l_class = l.class
+        r_class = r.class
+
+        # equal pairs: merge. ----------------------------------------------------
+        if l_class == r_class
+          return l + r unless l_class == Hash
+
+          warn_about_duplicate_keys(l, r)
+          return l.merge(r)
+
+        end
+
+        # Phase 43: Cache instance_of? checks to avoid repeated method calls
+        # unequal pairs: hoist to same level. ------------------------------------
+        l_is_slice = l.instance_of?(Parsanol::Slice)
+        r_is_slice = r.instance_of?(Parsanol::Slice)
+        l_is_str = l_class == String || l_is_slice
+        r_is_str = r_class == String || r_is_slice
+
+        # Maybe classes are not equal, but both are stringlike?
+        if l_is_str && r_is_str
+          # if we're merging a String with a Slice, the slice wins.
+          return r if r_is_slice
+          return l if l_is_slice
+
+          raise 'NOTREACHED: What other stringlike classes are there?'
+        end
+
+        # special case: If one of them is a string/slice, the other is more important
+        return l if r_is_str
+        return r if l_is_str
+
+        # otherwise just create an array for one of them to live in
+        return l + [r] if r_class == Hash
+        return [l] + r if l_class == Hash
+
+        raise "Unhandled case when foldr'ing sequence."
+      end
+
+      # Flatten results from a repetition of a single parslet. named indicates
+      # whether the user has named the result or not. If the user has named
+      # the results, we want to leave an empty list alone - otherwise it is
+      # turned into an empty string.
+      #
+      # @api private
+      #
+      # Phase 43: Optimized flatten_repetition - reduce array iterations
+      def flatten_repetition(list, named)
+        # Phase 43: Single pass to check for hashes and arrays
+        has_hash = false
+        has_array = false
+
+        i = 0
+        len = list.size
+        while i < len
+          e = list[i]
+          has_hash = true if e.instance_of?(Hash)
+          has_array = true if e.instance_of?(Array)
+          break if has_hash && has_array # Early exit if both found
+
+          i += 1
+        end
+
+        if has_hash
+          # If keyed subtrees are in the array, we'll want to discard all
+          # strings inbetween. To keep them, name them.
+          return list.select { |e| e.instance_of?(Hash) }
+        end
+
+        if has_array
+          # If any arrays are nested in this array, flatten all arrays to this
+          # level.
+          return list
+                 .select { |e| e.instance_of?(Array) }
+                 .flatten(1)
+        end
+
+        # Consistent handling of empty lists, when we act on a named result
+        return [] if named && list.empty?
+
+        # If there are only strings, concatenate them and return that.
+        foldl(list.compact) { |s, e| s + e }
+      end
+
+      # That annoying warning 'Duplicate subtrees while merging result' comes
+      # from here. You should add more '.as(...)' names to your intermediary tree.
+      #
+      def warn_about_duplicate_keys(h1, h2)
+        d = h1.keys & h2.keys
+        return if d.empty?
+
+        warn "Duplicate subtrees while merging result of \n  #{inspect}\nonly the values " \
+             "of the latter will be kept. (keys: #{d.inspect})"
+      end
+    end
+  end
+end

data/lib/parsanol/atoms/capture.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Captures the result of parsing and stores it for later use.
+# Use the capture method to capture a sub-expression result, then
+# access it via context.captures[:name] in dynamic blocks.
+#
+# @example
+#   str('a').capture(:first) >> dynamic { |ctx| str(ctx.captures[:first]) }
+#
+module Parsanol
+  module Atoms
+    class Capture < Parsanol::Atoms::Base
+      attr_reader :inner_atom, :capture_key
+
+      def initialize(atom, name)
+        super()
+        @inner_atom = atom
+        @capture_key = name.to_sym
+      end
+
+      def apply(source, context, consume_all)
+        success, result = @inner_atom.apply(source, context, consume_all)
+
+        if success
+          # Flatten and store the captured value in context
+          flattened = flatten(result)
+          context.captures[@capture_key] = flattened
+        end
+
+        [success, result]
+      end
+
+      def to_s_inner(prec)
+        "(#{@capture_key.inspect} = #{@inner_atom.to_s(prec)})"
+      end
+    end
+  end
+end