plurimath-parslet 3.0.0

data/lib/parslet/accelerator.rb

@@ -0,0 +1,162 @@
+
+
+# Optimizes the parsers by pattern matching on the parser atoms and replacing
+# matches with better versions. See the file qed/accelerators.md for a more
+# in-depth description.
+#
+# Example: 
+#   quote = str('"')
+#   parser = quote >> (quote.absent? >> any).repeat >> quote
+#
+#   A = Accelerator # for making what follows a bit shorter
+#   optimized_parser = A.apply(parser, 
+#     A.rule( (A.str(:x).absent? >> A.any).repeat ) { GobbleUp.new(x) })
+#
+#   optimized_parser.parse('"Parsing is now fully optimized! (tm)"')
+#
+module Parslet::Accelerator
+
+  # An expression to match against a tree of parser atoms. Normally, an
+  # expression is produced by Parslet::Accelerator.any, 
+  # Parslet::Accelerator.str or Parslet::Accelerator.re.
+  #
+  # Expressions can be chained much like parslet atoms can be: 
+  #
+  #   expr.repeat(1)      # matching repetition
+  #   expr.absent?        # matching absent?
+  #   expr.present?       # matching present?
+  #   expr1 >> expr2      # matching a sequence
+  #   expr1 | expr2       # matching an alternation
+  # 
+  # @see Parslet::Accelerator.str
+  # @see Parslet::Accelerator.re
+  # @see Parslet::Accelerator.any
+  #
+  # @see Parslet::Accelerator
+  # 
+  class Expression
+    attr_reader :type
+    attr_reader :args
+
+    def initialize(type, *args)
+      @type = type
+      @args = args
+    end
+
+    # @return [Expression]
+    def >> other_expr
+      join_or_new :seq, other_expr
+    end
+
+    # @return [Expression]
+    def | other_expr
+      join_or_new :alt, other_expr
+    end
+
+    # @return [Expression]
+    def absent?
+      Expression.new(:absent, self)
+    end
+    # @return [Expression]
+    def present?
+      Expression.new(:present, self)
+    end
+
+    # @return [Expression]
+    def repeat min=0, max=nil
+      Expression.new(:rep, min, max, self)
+    end
+
+    # @return [Expression]
+    def as name
+      Expression.new(:as, name)
+    end
+
+    # @api private
+    # @return [Expression]
+    def join_or_new tag, other_expr
+      if type == tag
+        @args << other_expr
+        self
+      else
+        Expression.new(tag, self, other_expr)
+      end
+    end
+  end
+
+module_function 
+  # Returns a match expression that will match `str` parslet atoms.
+  #
+  # @return [Parslet::Accelerator::Expression]
+  #
+  def str variable, *constraints
+    Expression.new(:str, variable, *constraints)
+  end
+
+  # Returns a match expression that will match `match` parslet atoms.
+  #
+  # @return [Parslet::Accelerator::Expression]
+  #
+  def re variable, *constraints
+    Expression.new(:re, variable, *constraints)
+  end
+
+  # Returns a match expression that will match `any` parslet atoms.
+  #
+  # @return [Parslet::Accelerator::Expression]
+  #
+  def any
+    Expression.new(:re, ".")
+  end
+
+  # Given a parslet atom and an expression, will determine if the expression
+  # matches the atom. If successful, returns the bindings into the pattern
+  # that were made. If no bindings had to be made to make the match successful, 
+  # the empty hash is returned. 
+  #
+  # @param atom [Parslet::Atoms::Base] parslet atom to match against
+  # @param expr [Parslet::Accelerator::Expression] expression to match
+  # @return [nil, Hash] bindings for the match, nil on failure
+  #
+  def match atom, expr
+    engine = Engine.new
+
+    return engine.bindings if engine.match(atom, expr)
+  end
+
+  # Constructs an accelerator rule. A rule is a matching expression and the
+  # code that should be executed once the expression could be bound to a 
+  # parser. 
+  #
+  # Example: 
+  #   Accelerator.rule(Accelerator.any) { Parslet.match('.') }
+  #
+  def rule expression, &action
+    [expression, action]
+  end
+
+  # Given a parslet atom and a set of rules, tries to match the rules 
+  # recursively through the parslet atom. Once a rule could be matched, 
+  # its action block will be called.
+  #
+  # Example: 
+  #   quote = str('"')
+  #   parser = quote >> (quote.absent? >> any).repeat >> quote
+  #
+  #   A = Accelerator # for making what follows a bit shorter
+  #   optimized_parser = A.apply(parser, 
+  #     A.rule( (A.str(:x).absent? >> A.any).repeat ) { GobbleUp.new(x) })
+  #
+  #   optimized_parser.parse('"Parsing is now fully optimized! (tm)"')
+  #
+  # @param atom [Parslet::Atoms::Base] a parser to optimize
+  # @param *rules [Parslet::Accelerator::Rule] rules produced by .rule
+  # @return [Parslet::Atoms::Base] optimized parser
+  #
+  def apply atom, *rules
+    Application.new(atom, rules).call
+  end
+end
+
+require 'parslet/accelerator/engine'
+require 'parslet/accelerator/application'

data/lib/parslet/atoms/alternative.rb

@@ -0,0 +1,53 @@
+
+# Alternative during matching. Contains a list of parslets that is tried each
+# one in turn. Only fails if all alternatives fail. 
+#
+# Example: 
+# 
+#   str('a') | str('b')   # matches either 'a' or 'b'
+#
+class Parslet::Atoms::Alternative < Parslet::Atoms::Base
+  attr_reader :alternatives
+  
+  # Constructs an Alternative instance using all given parslets in the order
+  # given. This is what happens if you call '|' on existing parslets, like 
+  # this: 
+  #
+  #   str('a') | str('b')
+  #
+  def initialize(*alternatives)
+    super()
+    
+    @alternatives = alternatives
+  end
+
+  #---
+  # Don't construct a hanging tree of Alternative parslets, instead store them
+  # all here. This reduces the number of objects created.
+  #+++
+  def |(parslet)
+    self.class.new(*@alternatives + [parslet])
+  end
+
+  def error_msg
+    @error_msg ||= "Expected one of #{alternatives.inspect}"
+  end
+
+  def try(source, context, consume_all)
+    errors = alternatives.map { |a|
+      success, value = result = a.apply(source, context, consume_all)
+      return result if success
+      
+      # Aggregate all errors
+      value
+    }
+    
+    # If we reach this point, all alternatives have failed. 
+    context.err(self, source, error_msg, errors)
+  end
+
+  precedence ALTERNATE
+  def to_s_inner(prec)
+    alternatives.map { |a| a.to_s(prec) }.join(' / ')
+  end
+end

data/lib/parslet/atoms/base.rb

@@ -0,0 +1,157 @@
+# Base class for all parslets, handles orchestration of calls and implements
+# a lot of the operator and chaining methods.
+#
+# Also see Parslet::Atoms::DSL chaining parslet atoms together.
+#
+class Parslet::Atoms::Base
+  include Parslet::Atoms::Precedence
+  include Parslet::Atoms::DSL
+  include Parslet::Atoms::CanFlatten
+
+  # Parslet label as provided in grammar
+  attr_accessor :label
+
+  # Given a string or an IO object, this will attempt a parse of its contents
+  # and return a result. If the parse fails, a Parslet::ParseFailed exception
+  # will be thrown. 
+  #
+  # @param io [String, Source] input for the parse process
+  # @option options [Parslet::ErrorReporter] :reporter error reporter to use, 
+  #   defaults to Parslet::ErrorReporter::Tree 
+  # @option options [Boolean] :prefix Should a prefix match be accepted? 
+  #   (default: false)
+  # @return [Hash, Array, Parslet::Slice] PORO (Plain old Ruby object) result
+  #   tree
+  #
+  def parse(io, options={})
+    source = io.respond_to?(:line_and_column) ? 
+      io : 
+      Parslet::Source.new(io)
+
+    # Try to cheat. Assuming that we'll be able to parse the input, don't 
+    # run error reporting code. 
+    success, value = setup_and_apply(source, nil, !options[:prefix])
+    
+    # If we didn't succeed the parse, raise an exception for the user. 
+    # Stack trace will be off, but the error tree should explain the reason
+    # it failed.
+    unless success
+      # Cheating has not paid off. Now pay the cost: Rerun the parse,
+      # gathering error information in the process.
+      reporter = options[:reporter] || Parslet::ErrorReporter::Tree.new
+      source.bytepos = 0
+      success, value = setup_and_apply(source, reporter, !options[:prefix])
+      
+      fail "Assertion failed: success was true when parsing with reporter" \
+        if success
+      
+      # Value is a Parslet::Cause, which can be turned into an exception:
+      value.raise
+      
+      fail "NEVER REACHED"
+    end
+    
+    # assert: success is true
+
+    # Extra input is now handled inline with the rest of the parsing. If 
+    # really we have success == true, prefix: false and still some input 
+    # is left dangling, that is a BUG.
+    if !options[:prefix] && source.chars_left > 0
+      fail "BUG: New error strategy should not reach this point."
+    end
+    
+    return flatten(value)
+  end
+  
+  # Creates a context for parsing and applies the current atom to the input. 
+  # Returns the parse result. 
+  #
+  # @return [<Boolean, Object>] Result of the parse. If the first member is 
+  #   true, the parse has succeeded. 
+  def setup_and_apply(source, error_reporter, consume_all)
+    context = Parslet::Atoms::Context.new(error_reporter)
+    apply(source, context, consume_all)
+  end
+
+  # Calls the #try method of this parslet. Success consumes input, error will 
+  # rewind the input. 
+  #
+  # @param source [Parslet::Source] source to read input from
+  # @param context [Parslet::Atoms::Context] context to use for the parsing
+  # @param consume_all [Boolean] true if the current parse must consume
+  #   all input by itself.
+  def apply(source, context, consume_all=false)
+    old_pos = source.bytepos
+    
+    success, _ = result = context.try_with_cache(self, source, consume_all)
+
+    if success
+      # Notify context
+      context.succ(source)
+      # If a consume_all parse was made and doesn't result in the consumption
+      # of all the input, that is considered an error. 
+      if consume_all && source.chars_left>0
+        # Read 10 characters ahead. Why ten? I don't know. 
+        offending_pos   = source.pos
+        offending_input = source.consume(10)
+        
+        # Rewind input (as happens always in error case)
+        source.bytepos  = old_pos
+        
+        return context.err_at(
+          self, 
+          source, 
+          "Don't know what to do with #{offending_input.to_s.inspect}", 
+          offending_pos
+        ) 
+      end
+      
+      # Looks like the parse was successful after all. Don't rewind the input.
+      return result
+    end
+    
+    # We only reach this point if the parse has failed. Rewind the input.
+    source.bytepos = old_pos
+    return result
+  end
+  
+  # Override this in your Atoms::Base subclasses to implement parsing
+  # behaviour. 
+  #
+  def try(source, context, consume_all)
+    raise NotImplementedError, \
+      "Atoms::Base doesn't have behaviour, please implement #try(source, context)."
+  end
+
+  # Returns true if this atom can be cached in the packrat cache. Most parslet
+  # atoms are cached, so this always returns true, unless overridden.
+  #
+  def cached?
+    true
+  end
+
+  # Debug printing - in Treetop syntax. 
+  #
+  def self.precedence(prec)
+    define_method(:precedence) { prec }
+  end
+  precedence BASE
+  def to_s(outer_prec=OUTER)
+    str = label || to_s_inner(precedence)
+    if outer_prec < precedence
+      "(#{str})"
+    else
+      str
+    end
+  end
+  def inspect
+    to_s(OUTER)
+  end
+private
+
+  # Produces an instance of Success and returns it. 
+  #
+  def succ(result)
+    [true, result]
+  end
+end

data/lib/parslet/atoms/can_flatten.rb

@@ -0,0 +1,137 @@
+
+module Parslet::Atoms
+  # A series of helper functions that have the common topic of flattening 
+  # result values into the intermediary tree that consists of Ruby Hashes and 
+  # Arrays. 
+  #
+  # This module has one main function, #flatten, that takes an annotated 
+  # structure as input and returns the reduced form that users expect from 
+  # Atom#parse. 
+  #
+  # NOTE: Since all of these functions are just that, functions without 
+  # side effects, they are in a module and not in a class. Its hard to draw 
+  # the line sometimes, but this is beyond. 
+  #
+  module CanFlatten
+    # Takes a mixed value coming out of a parslet and converts it to a return
+    # value for the user by dropping things and merging hashes. 
+    #
+    # Named is set to true if this result will be embedded in a Hash result from 
+    # naming something using <code>.as(...)</code>. It changes the folding 
+    # semantics of repetition.
+    #
+    def flatten(value, named=false)
+      # Passes through everything that isn't an array of things
+      return value unless value.instance_of? Array
+
+      # Extracts the s-expression tag
+      tag, *tail = value
+
+      # Merges arrays:
+      result = tail.
+        map { |e| flatten(e) }            # first flatten each element
+
+      case tag
+        when :sequence
+          return flatten_sequence(result)
+        when :maybe
+          return named ? result.first : result.first || ''
+        when :repetition
+          return flatten_repetition(result, named)
+      end
+
+      fail "BUG: Unknown tag #{tag.inspect}."
+    end
+
+    # Lisp style fold left where the first element builds the basis for 
+    # an inject. 
+    #
+    def foldl(list, &block)
+      return '' if list.empty?
+      list[1..-1].inject(list.first, &block)
+    end
+
+    # Flatten results from a sequence of parslets. 
+    #
+    # @api private
+    #
+    def flatten_sequence(list)
+      foldl(list.compact) { |r, e|        # and then merge flat elements
+        merge_fold(r, e)
+      }
+    end
+    # @api private 
+    def merge_fold(l, r)
+      # equal pairs: merge. ----------------------------------------------------
+      if l.class == r.class
+        if l.is_a?(Hash)
+          warn_about_duplicate_keys(l, r)
+          return l.merge(r)
+        else
+          return l + r
+        end
+      end
+
+      # unequal pairs: hoist to same level. ------------------------------------
+
+      # Maybe classes are not equal, but both are stringlike?
+      if l.respond_to?(:to_str) && r.respond_to?(:to_str)
+        # if we're merging a String with a Slice, the slice wins. 
+        return r if r.respond_to? :to_slice
+        return l if l.respond_to? :to_slice
+
+        fail "NOTREACHED: What other stringlike classes are there?"
+      end
+
+      # special case: If one of them is a string/slice, the other is more important 
+      return l if r.respond_to? :to_str
+      return r if l.respond_to? :to_str
+
+      # otherwise just create an array for one of them to live in 
+      return l + [r] if r.class == Hash
+      return [l] + r if l.class == Hash
+
+      fail "Unhandled case when foldr'ing sequence."
+    end
+
+    # Flatten results from a repetition of a single parslet. named indicates
+    # whether the user has named the result or not. If the user has named
+    # the results, we want to leave an empty list alone - otherwise it is 
+    # turned into an empty string. 
+    #
+    # @api private
+    #
+    def flatten_repetition(list, named)
+      if list.any? { |e| e.instance_of?(Hash) }
+        # If keyed subtrees are in the array, we'll want to discard all 
+        # strings inbetween. To keep them, name them. 
+        return list.select { |e| e.instance_of?(Hash) }
+      end
+
+      if list.any? { |e| e.instance_of?(Array) }
+        # If any arrays are nested in this array, flatten all arrays to this
+        # level. 
+        return list.
+          select { |e| e.instance_of?(Array) }.
+          flatten(1)
+      end
+
+      # Consistent handling of empty lists, when we act on a named result        
+      return [] if named && list.empty?
+
+      # If there are only strings, concatenate them and return that. 
+      foldl(list.compact) { |s,e| s+e }
+    end
+
+    # That annoying warning 'Duplicate subtrees while merging result' comes 
+    # from here. You should add more '.as(...)' names to your intermediary tree.
+    #
+    def warn_about_duplicate_keys(h1, h2)
+      d = h1.keys & h2.keys
+      unless d.empty?
+        warn "Duplicate subtrees while merging result of \n  #{self.inspect}\nonly the values"+
+             " of the latter will be kept. (keys: #{d.inspect})"
+      end
+    end
+  end
+end

data/lib/parslet/atoms/capture.rb

@@ -0,0 +1,38 @@
+
+# Stores the result of matching an atom against input in the #captures in 
+# parse context. Doing so will allow you to pull parts of the ongoing parse
+# out later and use them to match other pieces of input. 
+#
+# Example: 
+#   # After this, context.captures[:an_a] returns 'a'
+#   str('a').capture(:an_a)
+#
+#   # Capture and use of the capture: (matches either 'aa' or 'bb')
+#   match['ab'].capture(:first) >> 
+#     dynamic { |src, ctx| str(ctx.captures[:first]) }
+#   
+class Parslet::Atoms::Capture < Parslet::Atoms::Base
+  attr_reader :parslet, :name
+
+  def initialize(parslet, name)
+    super()
+
+    @parslet, @name = parslet, name
+  end
+
+  def apply(source, context, consume_all)
+    success, value = result = parslet.apply(source, context, consume_all)
+
+    if success
+      context.captures[name.to_sym] = 
+        flatten(value)
+    end
+    
+    return result
+  end
+  
+  def to_s_inner(prec)
+    "(#{name.inspect} = #{parslet.to_s(prec)})"
+  end
+end
+

data/lib/parslet/atoms/context.rb

@@ -0,0 +1,103 @@
+module Parslet::Atoms
+  # Helper class that implements a transient cache that maps position and
+  # parslet object to results. This is used for memoization in the packrat
+  # style. 
+  #
+  # Also, error reporter is stored here and error reporting happens through
+  # this class. This makes the reporting pluggable. 
+  #
+  class Context
+    # @param reporter [#err, #err_at] Error reporter (leave empty for default 
+    #   reporter)
+    def initialize(reporter=Parslet::ErrorReporter::Tree.new)
+      @cache = Hash.new { |h, k| h[k] = {} }
+      @reporter = reporter
+      @captures = Parslet::Scope.new
+    end
+    
+    # Caches a parse answer for obj at source.pos. Applying the same parslet
+    # at one position of input always yields the same result, unless the input
+    # has changed. 
+    #
+    # We need the entire source here so we can ask for how many characters 
+    # were consumed by a successful parse. Imitation of such a parse must 
+    # advance the input pos by the same amount of bytes.
+    #
+    def try_with_cache(obj, source, consume_all)
+      beg = source.bytepos
+        
+      # Not in cache yet? Return early.
+      unless entry = lookup(obj, beg)
+        result = obj.try(source, self, consume_all)
+    
+        if obj.cached?
+          set obj, beg, [result, source.bytepos-beg]
+        end
+        
+        return result
+      end
+
+      # the condition in unless has returned true, so entry is not nil.
+      result, advance = entry
+
+      # The data we're skipping here has been read before. (since it is in 
+      # the cache) PLUS the actual contents are not interesting anymore since
+      # we know obj matches at beg. So skip reading.
+      source.bytepos = beg + advance
+      return result
+    end  
+    
+    # Report an error at a given position. 
+    # @see ErrorReporter
+    #
+    def err_at(*args)
+      return [false, @reporter.err_at(*args)] if @reporter
+      return [false, nil]
+    end
+    
+    # Report an error. 
+    # @see ErrorReporter
+    #
+    def err(*args)
+      return [false, @reporter.err(*args)] if @reporter
+      return [false, nil]
+    end
+
+    # Report a successful parse.
+    # @see ErrorReporter::Contextual
+    #
+    def succ(*args)
+      return [true, @reporter.succ(*args)] if @reporter
+      return [true, nil]
+    end
+
+    # Returns the current captures made on the input (see
+    # Parslet::Atoms::Base#capture). Use as follows: 
+    # 
+    #   context.captures[:foobar] # => returns capture :foobar
+    #
+    attr_reader :captures
+    
+    # Starts a new scope. Use the #scope method of Parslet::Atoms::DSL
+    # to call this. 
+    #
+    def scope
+      captures.push
+      yield
+    ensure
+      captures.pop
+    end
+    
+  private 
+    # NOTE These methods use #object_id directly, since that seems to bring the
+    # most performance benefit. This is a hot spot; going through
+    # Atoms::Base#hash doesn't yield as much.
+    #
+    def lookup(obj, pos)
+      @cache[pos][obj.object_id] 
+    end
+    def set(obj, pos, val)
+      @cache[pos][obj.object_id] = val
+    end
+  end
+end