parsanol 3.0.0

data/lib/parsanol/parser.rb

@@ -0,0 +1,151 @@
+
+# The base class for all your parsers. Use as follows: 
+#
+#   require 'parslet'
+#        
+#   class MyParser < Parsanol::Parser
+#     rule(:a) { str('a').repeat }
+#     root(:a)        
+#   end
+#        
+#   pp MyParser.new.parse('aaaa')   # => 'aaaa'
+#   pp MyParser.new.parse('bbbb')   # => Parsanol::Atoms::ParseFailed: 
+#                                   #    Don't know what to do with bbbb at line 1 char 1.
+#
+# Parsanol::Parser is also a grammar atom. This means that you can mix full 
+# fledged parsers freely with small parts of a different parser. 
+#
+# Example: 
+#   class ParserA < Parsanol::Parser
+#     root :aaa
+#     rule(:aaa) { str('a').repeat(3,3) }
+#   end
+#   class ParserB < Parsanol::Parser
+#     root :expression
+#     rule(:expression) { str('b') >> ParserA.new >> str('b') }
+#   end
+#
+# In the above example, ParserB would parse something like 'baaab'. 
+#
+class Parsanol::Parser < Parsanol::Atoms::Base
+  include Parsanol
+
+  class << self # class methods
+    # Define the parsers #root function. This is the place where you start 
+    # parsing; if you have a rule for 'file' that describes what should be 
+    # in a file, this would be your root declaration: 
+    #
+    #   class Parser
+    #     root :file
+    #     rule(:file) { ... }
+    #   end
+    #
+    # #root declares a 'parse' function that works just like the parse 
+    # function that you can call on a simple parslet, taking a string as input
+    # and producing parse output. 
+    #
+    # In a way, #root is a shorthand for: 
+    #
+    #   def parse(str)
+    #     your_parser_root.parse(str)
+    #   end
+    #
+    def root(name)
+      undef_method :root if method_defined? :root
+      define_method(:root) do
+        self.send(name)
+      end
+    end
+  end
+  
+  def try(source, context, consume_all)
+    root.try(source, context, consume_all)
+  end
+
+  def to_s_inner(prec)
+    root.to_s(prec)
+  end
+
+  # Unified parsing method with mode selection
+  #
+  # @param input [String] Input string to parse
+  # @param mode_or_options [Symbol, Hash] Parsing mode (:ruby, :native, :json) or options hash
+  # @param options [Hash] Additional options (if mode provided):
+  #   - :reporter - Error reporter to use
+  #   - :prefix - Allow prefix matching (default: false)
+  #
+  # @return [Hash, Array, String, Parsanol::Slice] Parsed result
+  # @raise [Parsanol::ParseFailed] If parsing fails
+  #
+  # @example Parse in Ruby mode (default)
+  #   parser.parse("1+2")
+  #   # => {:left=>"1", :op=>"+", :right=>"2"}
+  #
+  # @example Parse in native mode (faster, if extension available)
+  #   parser.parse("1+2", mode: :native)
+  #   # => {:left=>"1", :op=>"+", :right=>"2"}
+  #
+  # @example Parse in JSON mode (for cross-language use)
+  #   parser.parse("1+2", mode: :json)
+  #   # => '{"left":"1","op":"+","right":"2"}'
+  #
+  def parse(input, mode_or_options = {}, **options)
+    # Support both old API (parse(input, options)) and new API (parse(input, mode: :ruby))
+    if mode_or_options.is_a?(Hash)
+      # Old API: parse(input, options={})
+      # Merge keyword options into the options hash (handles parse(str, prefix: true))
+      merged_options = mode_or_options.merge(options)
+      super(input, merged_options)
+    else
+      # New API: parse(input, mode: :ruby, **options)
+      mode = mode_or_options
+      case mode
+      when :ruby
+        parse_ruby(input, options)
+      when :native
+        parse_native(input, options)
+      when :json
+        parse_json(input, options)
+      else
+        raise ArgumentError, "Unknown mode: #{mode}. Use :ruby, :native, or :json"
+      end
+    end
+  end
+
+  # Batch parsing with mode selection
+  #
+  # @param inputs [Array<String>] Array of input strings
+  # @param mode [Symbol] Parsing mode (:ruby, :native, or :json)
+  # @param options [Hash] Additional options
+  # @return [Array] Array of parsed results
+  #
+  def parse_batch(inputs, mode: :ruby, **options)
+    inputs.map { |input| parse(input, mode: mode, **options) }
+  end
+
+  private
+
+  def parse_ruby(input, options = {})
+    # Use the original Ruby parser from Base
+    super(input, options)
+  end
+
+  def parse_native(input, options = {})
+    # Use native if available, fallback to pure Ruby
+    if Parsanol::Native.available?
+      Parsanol::Native.parse_parslet_compatible(root, input)
+    else
+      parse_ruby(input, options)
+    end
+  end
+
+  def parse_json(input, options = {})
+    if Parsanol::Native.available?
+      grammar_json = Parsanol::Native.serialize_grammar(root)
+      result = Parsanol::Native.parse(grammar_json, input)
+      result.to_json
+    else
+      parse_ruby(input, options).to_json
+    end
+  end
+end

data/lib/parsanol/parslet.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+# Parsanol::Parslet - Nested compatibility layer for original Parslet API
+#
+# This provides backwards compatibility for code that uses the original Parslet API.
+# Instead of root-level Parslet constant, we use Parsanol::Parslet as a nested module.
+#
+# Usage:
+#   require 'parsanol/parslet'
+#
+#   class MyParser < Parsanol::Parslet::Parser
+#     include Parsanol::Parslet
+#     rule(:foo) { str('foo') }
+#     root(:foo)
+#   end
+#
+# Migration from original Parslet:
+#   Before: require 'parslet'
+#           class MyParser < Parslet::Parser
+#           include Parslet
+#
+#   After:  require 'parsanol/parslet'
+#           class MyParser < Parsanol::Parslet::Parser
+#           include Parsanol::Parslet
+
+require 'parsanol'
+
+module Parsanol
+  module Parslet
+    # Include Parsanol to get all DSL methods (str, match, any, etc.)
+    include Parsanol
+
+    # Error class alias for compatibility
+    ParseFailed = Parsanol::ParseFailed
+
+    # Atoms namespace - aliases to Parsanol atoms
+    # These are the atoms explicitly loaded by lib/parsanol/atoms.rb
+    module Atoms
+      Base = ::Parsanol::Atoms::Base
+      Str = ::Parsanol::Atoms::Str
+      Re = ::Parsanol::Atoms::Re
+      Sequence = ::Parsanol::Atoms::Sequence
+      Alternative = ::Parsanol::Atoms::Alternative
+      Repetition = ::Parsanol::Atoms::Repetition
+      Named = ::Parsanol::Atoms::Named
+      Entity = ::Parsanol::Atoms::Entity
+      Lookahead = ::Parsanol::Atoms::Lookahead
+      Cut = ::Parsanol::Atoms::Cut
+      Capture = ::Parsanol::Atoms::Capture
+      Scope = ::Parsanol::Atoms::Scope
+      Dynamic = ::Parsanol::Atoms::Dynamic
+      Infix = ::Parsanol::Atoms::Infix
+      Ignored = ::Parsanol::Atoms::Ignored
+      ParseFailed = ::Parsanol::ParseFailed
+    end
+
+    # Class aliases
+    Parser = ::Parsanol::Parser
+    Transform = ::Parsanol::Transform
+    Cause = ::Parsanol::Cause
+    Slice = ::Parsanol::Slice
+    Source = ::Parsanol::Source
+    Pattern = ::Parsanol::Pattern
+    Context = ::Parsanol::Context
+
+    # Module functions for DSL (delegate to Parsanol)
+    extend self
+
+    def match(str = nil)
+      Parsanol.match(str)
+    end
+
+    def str(str)
+      Parsanol.str(str)
+    end
+
+    def any
+      Parsanol.any
+    end
+
+    def scope(&block)
+      Parsanol.scope(&block)
+    end
+
+    def dynamic(&block)
+      Parsanol.dynamic(&block)
+    end
+
+    def infix_expression(element, *operations, &reducer)
+      Parsanol.infix_expression(element, *operations, &reducer)
+    end
+
+    def exp(str)
+      Parsanol.exp(str)
+    end
+
+    def sequence(symbol)
+      Parsanol.sequence(symbol)
+    end
+
+    def simple(symbol)
+      Parsanol.simple(symbol)
+    end
+
+    def subtree(symbol)
+      Parsanol.subtree(symbol)
+    end
+
+    # Class method extensions for Parser
+    module ClassMethods
+      # Delegate rule definition to Parsanol's implementation
+      # This works for both classes and modules that include Parsanol::Parslet
+      def rule(name, opts = {}, &definition)
+        # Remove existing method if present
+        undef_method name if method_defined? name
+
+        # Define the rule method that memoizes the entity
+        define_method(name) do
+          @rules ||= {}     # <name, rule> memoization
+          return @rules[name] if @rules.has_key?(name)
+
+          # Capture the self of the parser class along with the definition.
+          definition_closure = proc {
+            result = instance_eval(&definition)
+
+            # Apply optimizations if enabled (only for classes that support it)
+            if self.class.respond_to?(:optimize_rules?) && self.class.optimize_rules?
+              # Apply all optimizers: quantifiers, sequences, choices, and lookaheads
+              result = Parsanol::Optimizer.simplify_quantifiers(result)
+              result = Parsanol::Optimizer.simplify_sequences(result)
+              result = Parsanol::Optimizer.simplify_choices(result)
+              result = Parsanol::Optimizer.simplify_lookaheads(result)
+            end
+
+            result
+          }
+
+          @rules[name] = Parsanol::Atoms::Entity.new(name, opts[:label], &definition_closure)
+        end
+      end
+    end
+
+    # Extend with class methods when included
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+  end
+end

data/lib/parsanol/pattern/binding.rb

@@ -0,0 +1,49 @@
+
+# Used internally for representing a bind placeholder in a Parsanol::Transform
+# pattern. This is the superclass for all bindings. 
+#
+# It defines the most permissive kind of bind, the one that matches any subtree
+# whatever it looks like. 
+#
+class Parsanol::Pattern::SubtreeBind < Struct.new(:symbol)
+  def variable_name
+    symbol
+  end
+  
+  def inspect
+    "#{bind_type_name}(#{symbol.inspect})"
+  end
+  
+  def can_bind?(subtree)
+    true
+  end
+
+private 
+  def bind_type_name 
+    if md=self.class.name.match(/(\w+)Bind/)
+      md.captures.first.downcase
+    else
+      # This path should never be used, but since this is for inspection only, 
+      # let's not raise.
+      'unknown_bind'
+    end
+  end
+end
+
+# Binds a symbol to a simple subtree, one that is not either a sequence of
+# elements or a collection of attributes. 
+#
+class Parsanol::Pattern::SimpleBind < Parsanol::Pattern::SubtreeBind
+  def can_bind?(subtree)
+    not [Hash, Array].include?(subtree.class)
+  end
+end
+
+# Binds a symbol to a sequence of simple leafs ([element1, element2, ...])
+#
+class Parsanol::Pattern::SequenceBind < Parsanol::Pattern::SubtreeBind
+  def can_bind?(subtree)
+    subtree.kind_of?(Array) &&
+      (not subtree.any? { |el| [Hash, Array].include?(el.class) })
+  end
+end

data/lib/parsanol/pattern.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+# Matches trees against expressions. Trees are formed by arrays and hashes
+# for expressing membership and sequence. The leafs of the tree are other
+# classes. 
+#
+# A tree issued by the parslet library might look like this: 
+#
+#   { 
+#     :function_call => {
+#       :name => 'foobar', 
+#       :args => [1, 2, 3]
+#     }
+#   }
+#
+# A pattern that would match against this tree would be: 
+#
+#   { :function_call => { :name => simple(:name), :args => sequence(:args) }}
+#
+# Note that Parsanol::Pattern only matches at a given subtree; it wont try 
+# to match recursively. To do that, please use Parsanol::Transform. 
+#
+class Parsanol::Pattern  
+  def initialize(pattern)
+    @pattern = pattern
+  end
+
+  # Decides if the given subtree matches this pattern. Returns the bindings
+  # made on a successful match or nil if the match fails. If you specify 
+  # bindings to be a hash, the mappings in it will be treated like bindings
+  # made during an attempted match. 
+  #
+  #   Pattern.new('a').match('a', :foo => 'bar') # => { :foo => 'bar' }
+  #
+  # @param subtree [String, Hash, Array] poro subtree returned by a parse
+  # @param bindings [Hash] variable bindings to be verified
+  # @return [Hash, nil] On success: variable bindings that allow a match. On 
+  #   failure: nil
+  #
+  def match(subtree, bindings=nil)
+    bindings = bindings && bindings.dup || Hash.new
+    return bindings if element_match(subtree, @pattern, bindings)
+  end
+  
+  # Returns true if the tree element given by +tree+ matches the expression
+  # given by +exp+. This match must respect bindings already made in
+  # +bindings+. Note that bindings is carried along and modified. 
+  #
+  # @api private
+  #
+  def element_match(tree, exp, bindings) 
+    # p [:elm, tree, exp]
+    if tree.is_a?(Hash) && exp.is_a?(Hash)
+      return element_match_hash(tree, exp, bindings)
+    elsif tree.is_a?(Array) && exp.is_a?(Array)
+      return element_match_ary_single(tree, exp, bindings)
+    else
+      # If elements match exactly, then that is good enough in all cases
+      return true if exp === tree
+      
+      # If exp is a bind variable: Check if the binding matches
+      if exp.respond_to?(:can_bind?) && exp.can_bind?(tree)
+        return element_match_binding(tree, exp, bindings)
+      end
+                  
+      # Otherwise: No match (we don't know anything about the element
+      # combination)
+      return false
+    end
+  end
+  
+  # @api private
+  #
+  def element_match_binding(tree, exp, bindings)
+    var_name = exp.variable_name
+
+    # TODO test for the hidden :_ feature.
+    if var_name && bound_value = bindings[var_name]
+      return bound_value == tree
+    end
+    
+    # New binding: 
+    bindings.store var_name, tree
+    
+    return true
+  end
+  
+  # @api private
+  #
+  def element_match_ary_single(sequence, exp, bindings)
+    return false if sequence.size != exp.size
+    
+    return sequence.zip(exp).all? { |elt, subexp|
+      element_match(elt, subexp, bindings) }
+  end
+  
+  # @api private
+  #
+  def element_match_hash(tree, exp, bindings)
+    # Early failure when one hash is bigger than the other
+    return false unless exp.size == tree.size
+    
+    # We iterate over expected pattern, since we demand that the keys that
+    # are there should be in tree as well.
+    exp.each do |expected_key, expected_value|
+      return false unless tree.has_key? expected_key
+      
+      # Recurse into the value and stop early on failure
+      value = tree[expected_key]
+      return false unless element_match(value, expected_value, bindings)
+    end
+    
+    return true
+  end  
+end

data/lib/parsanol/pool.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Generic object pool for reducing garbage collection pressure.
+  #
+  # The ObjectPool class implements a simple object pooling strategy:
+  # - Objects are pre-allocated on initialization
+  # - Objects are reused instead of created new
+  # - Objects are reset before being returned to the pool
+  # - Pool size is bounded to prevent unbounded growth
+  #
+  # This reduces GC pressure by reusing objects instead of constantly
+  # creating and destroying them, which is particularly beneficial for
+  # frequently allocated objects like Slice instances.
+  #
+  # == Thread Safety
+  #
+  # This implementation is NOT thread-safe. If thread safety is required,
+  # wrap pool operations in a mutex or use thread-local pools.
+  #
+  # == Usage Example
+  #
+  #   # Create a pool for Slice objects
+  #   pool = Parsanol::ObjectPool.new(Parsanol::Slice, size: 1000)
+  #
+  #   # Acquire an object from the pool
+  #   slice = pool.acquire
+  #   slice.instance_variable_set(:@bytepos, 0)
+  #   slice.instance_variable_set(:@str, "hello")
+  #
+  #   # Use the slice...
+  #
+  #   # Return it to the pool for reuse
+  #   pool.release(slice)
+  #
+  # == Object Reset Protocol
+  #
+  # Objects returned to the pool will have their reset! method called
+  # if they respond to it. This allows objects to clean up their state
+  # before being reused. If reset! is not defined, the object is still
+  # pooled but without automatic cleanup.
+  #
+  class ObjectPool
+    # @return [Integer] Maximum number of objects to keep in the pool
+    attr_reader :size
+
+    # @return [Hash] Statistics about pool usage
+    attr_reader :stats
+
+    # Initialize a new object pool.
+    #
+    # @param klass [Class] The class of objects to pool
+    # @param size [Integer] Maximum number of objects to keep in pool (default: 1000)
+    # @param preallocate [Boolean] Whether to pre-allocate objects on initialization (default: true)
+    #
+    # @example Create a pool with default settings
+    #   pool = ObjectPool.new(Array, size: 1000)
+    #
+    # @example Create a pool without pre-allocation
+    #   pool = ObjectPool.new(Array, size: 1000, preallocate: false)
+    #
+    def initialize(klass, size: 1000, preallocate: true)
+      @klass = klass
+      @size = size
+      @available = []
+      @stats = {
+        created: 0,
+        reused: 0,
+        released: 0,
+        discarded: 0
+      }
+
+      # Pre-allocate objects for efficiency if requested
+      # This reduces allocation overhead during initial parsing
+      preallocate(size) if preallocate && can_preallocate?
+    end
+
+    # Acquire an object from the pool.
+    #
+    # If the pool has available objects, one is returned (and considered "reused").
+    # If the pool is empty, a new object is created (and considered "created").
+    #
+    # @return [Object] An object instance from the pool or newly created
+    #
+    # @example Acquire from pool
+    #   obj = pool.acquire
+    #
+    def acquire
+      if @available.empty?
+        @stats[:created] += 1
+        @klass.new
+      else
+        @stats[:reused] += 1
+        @available.pop
+      end
+    end
+
+    # Return an object to the pool for reuse.
+    #
+    # Before returning to the pool:
+    # 1. If object responds to reset!, that method is called to clean up state
+    # 2. If pool is at capacity, the object is discarded instead of pooled
+    #
+    # This ensures:
+    # - Objects are cleaned before reuse (no stale state)
+    # - Pool doesn't grow unbounded (respects size limit)
+    #
+    # @param obj [Object] The object to return to the pool
+    # @return [Boolean] true if object was returned to pool, false if discarded
+    #
+    # @example Return object to pool
+    #   pool.release(obj)
+    #
+    def release(obj)
+      # Don't pool if we're at capacity - discard instead
+      if @available.size >= @size
+        @stats[:discarded] += 1
+        return false
+      end
+
+      # Reset object state if it supports the protocol
+      obj.reset! if obj.respond_to?(:reset!)
+
+      @stats[:released] += 1
+      @available.push(obj)
+      true
+    end
+
+    # Get current pool statistics.
+    #
+    # Statistics include:
+    # - size: Maximum pool capacity
+    # - available: Number of objects currently available in pool
+    # - created: Total number of new objects created
+    # - reused: Total number of times objects were reused from pool
+    # - released: Total number of objects returned to pool
+    # - discarded: Total number of objects discarded (pool was full)
+    # - utilization: Percentage of acquires that were reused (0-100)
+    #
+    # @return [Hash] Hash containing pool statistics
+    #
+    # @example Get statistics
+    #   stats = pool.stats
+    #   puts "Pool utilization: #{stats[:utilization]}%"
+    #
+    def statistics
+      total_acquires = @stats[:created] + @stats[:reused]
+      utilization = total_acquires.zero? ? 0.0 : (@stats[:reused].to_f / total_acquires * 100)
+
+      {
+        size: @size,
+        available: @available.size,
+        created: @stats[:created],
+        reused: @stats[:reused],
+        released: @stats[:released],
+        discarded: @stats[:discarded],
+        utilization: utilization.round(2)
+      }
+    end
+
+    # Clear all objects from the pool.
+    #
+    # This removes all pooled objects and resets statistics.
+    # Useful for testing or when you want to force fresh allocations.
+    #
+    # @return [void]
+    #
+    # @example Clear the pool
+    #   pool.clear!
+    #
+    def clear!
+      @available.clear
+      @stats = {
+        created: 0,
+        reused: 0,
+        released: 0,
+        discarded: 0
+      }
+    end
+
+    private
+
+    # Check if the pooled class can be pre-allocated.
+    #
+    # Some classes require arguments to initialize and cannot be
+    # pre-allocated without those arguments. This method checks if
+    # the class has a zero-arity initialize method.
+    #
+    # @return [Boolean] true if class can be instantiated without arguments
+    #
+    def can_preallocate?
+      # Check if the class can be instantiated without arguments
+      # This is a heuristic - we try to create one instance to test
+      begin
+        @klass.new
+        true
+      rescue ArgumentError
+        # Class requires arguments, cannot pre-allocate
+        false
+      end
+    end
+
+    # Pre-allocate objects to fill the pool.
+    #
+    # This is called during initialization if preallocate: true is set.
+    # Pre-allocation reduces allocation overhead during initial parsing.
+    #
+    # @param count [Integer] Number of objects to pre-allocate
+    # @return [void]
+    #
+    def preallocate(count)
+      count.times do
+        @available.push(@klass.new)
+      end
+      # Adjust stats to reflect pre-allocation as "released" not "created"
+      # since these objects haven't been acquired yet
+      @stats[:released] = count
+    end
+  end
+end