parslet 1.4.0 → 1.5.0

data/HISTORY.txt

@@ -3,6 +3,22 @@
   - prsnt? and absnt? are now finally banned into oblivion. Wasting vocals for
     the win. 
     
+= 1.5 / ??
+    
+  + Handles unconsumed input at end of parse completely differently. Instead
+    of generating a toplevel error, it now raises an error in every branch
+    of the parse. More information in the resulting exception ensues! Thanks
+    again to John Mettraux for inspiration & acceptance specs. 
+    
+    NOTE that this means that the UnconsumedInput exception is gone, since the
+    unconsumed input case is nothing special anymore. 
+    
+  * This history now finally reads like the Changelog of the linux kernel. 
+    Meaning that probably no one ever reads this. 
+    
+  + Captures and parsing subsequent input based on captured values. This has
+    been long overdue - finally you can parse HEREdocs with parslet!
+    
 = 1.4.0 / 25May2012
 
   + Revised documentation. A few new API features have finally made it into

data/README

@@ -65,6 +65,6 @@ ruby-1.8.7-p334 for better results.
 
 STATUS 
 
-At version 1.4.0 - See HISTORY.txt for changes.
+Production worthy.
 
 (c) 2010, 2011, 2012 Kaspar Schiess

data/Rakefile

@@ -16,29 +16,10 @@ end
 
 task :default => :spec
 
-# Generate documentation
-RDoc::Task.new do |rdoc|
-  rdoc.title    = "parslet - construction of parsers made easy"
-  rdoc.options << '--line-numbers'
-  rdoc.options << '--fmt' << 'shtml' # explictly set shtml generator
-  rdoc.template = 'direct' # lighter template used on railsapi.com
-  rdoc.main = "README"
-  rdoc.rdoc_files.include("README", "lib/**/*.rb")
-  rdoc.rdoc_dir = "rdoc"
-end
-
-desc 'Clear out RDoc'
-task :clean => [:clobber_rdoc, :clobber_package]
-
 # This task actually builds the gem. 
 task :gem => :spec
 spec = eval(File.read('parslet.gemspec'))
 
-desc "Generate the gem package."
-Gem::PackageTask.new(spec) do |pkg|
-  pkg.gem_spec = spec
-end
-
 desc "Prints LOC stats"
 task :stat do
   %w(lib spec example).each do |dir|

data/example/calc.rb

@@ -0,0 +1,153 @@
+# A simple integer calculator to answer the question about how to do 
+# left and right associativity in parslet (PEG) once and for all. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'rspec'
+require 'parslet'
+require 'parslet/rig/rspec'
+
+# This is the parsing stage. It expresses left associativity by compiling
+# list of things that have the same associativity. 
+class CalcParser < Parslet::Parser
+  root :addition
+  
+  rule(:addition) {
+    multiplication.as(:l) >> (add_op >> multiplication.as(:r)).repeat(1) |
+    multiplication
+  }
+  
+  rule(:multiplication) { 
+    integer.as(:l) >> (mult_op >> integer.as(:r)).repeat(1) |
+    integer }
+  
+  rule(:integer) { digit.repeat(1).as(:i) >> space? }
+
+  rule(:mult_op) { match['*/'].as(:o) >> space? }
+  rule(:add_op) { match['+-'].as(:o) >> space? }
+
+  rule(:digit) { match['0-9'] }
+  rule(:space?) { match['\s'].repeat }
+end
+
+# Classes for the abstract syntax tree.
+Int    = Struct.new(:int) {
+  def eval; self end
+  def op(operation, other)
+    left = int
+    right = other.int 
+
+    Int.new(
+      case operation
+        when '+'
+          left + right
+        when '-'
+          left - right
+        when '*'
+          left * right
+        when '/'
+          left / right
+      end)
+  end
+  def to_i
+    int
+  end
+}
+Seq    = Struct.new(:sequence) {
+  def eval
+    sequence.reduce { |accum, operation| 
+      operation.call(accum) }
+  end
+}
+LeftOp = Struct.new(:operation, :right) {
+  def call(left)
+    left = left.eval
+    right = self.right.eval
+
+    left.op(operation, right)
+  end
+}
+
+# Transforming intermediary syntax tree into a real AST.
+class CalcTransform < Parslet::Transform
+  rule(i: simple(:i)) { Int.new(Integer(i)) }
+  rule(o: simple(:o), r: simple(:i)) { LeftOp.new(o, i) }
+  rule(l: simple(:i)) { i }
+  rule(sequence(:seq)) { Seq.new(seq) }
+end
+
+# And this calls everything in the right order.
+def calculate(str)
+  intermediary_tree = CalcParser.new.parse(str)
+  abstract_tree = CalcTransform.new.apply(intermediary_tree)
+  result = abstract_tree.eval
+  
+  result.to_i
+end
+
+# A test suite for the above parser
+describe CalcParser do
+  let(:p) { described_class.new }
+  describe '#integer' do
+    let(:i) { p.integer }
+    it "parses integers" do
+      i.should parse('1')
+      i.should parse('123')
+    end 
+    it "consumes trailing white space" do
+      i.should parse('123   ')
+    end 
+    it "doesn't parse floats" do
+      i.should_not parse('1.3')
+    end 
+  end
+  describe '#multiplication' do
+    let(:m) { p.multiplication }
+    it "parses simple multiplication" do
+      m.should parse('1*2')
+    end
+    it "parses division" do
+      m.should parse('1/2')
+    end 
+  end
+  describe '#addition' do
+    let(:a) { p.addition }
+    
+    it "parses simple addition" do
+      a.should parse('1+2')
+      a.should parse('1+2+3-4')
+    end 
+  end
+end
+describe CalcTransform do
+  def t(obj)
+    described_class.new.apply(obj)
+  end
+  
+  it "transforms integers" do
+    t(i: '1').should == Int.new(1)
+  end 
+  it "unwraps left operand" do
+    t(l: :obj).should == :obj
+  end 
+end
+describe 'whole computation specs' do
+  def self.result_of(str, int)
+    it(str) { calculate(str).should == int } 
+  end
+
+  result_of '1+1', 2
+  result_of '1-1-1', -1
+  result_of '1+1+3*5/2', 9
+  result_of '123*2', 246
+end
+
+
+# Enable these if you want to change the code.
+# RSpec::Core::Runner.run([], $stderr, $stdout)
+
+str = ARGV.join
+str = '123*2' if str.match(/^\s*$/)
+
+print "#{str} (command line): -> "
+puts calculate(str)

data/example/capture.rb

@@ -0,0 +1,49 @@
+
+# This example demonstrates how pieces of input can be captured and matched
+# against later on. Without this, you cannot match here-documents and other
+# self-dependent grammars. 
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'parslet'
+require 'parslet/convenience'
+require 'pp'
+
+class CapturingParser < Parslet::Parser
+  root :document
+  
+  # Introduce a scope for each document. This ensures that documents can be
+  # nested. 
+  rule(:document) { scope { doc_start >> text >> doc_end } }
+  
+  # Start of a document is a heredoc marker. This is captured in :marker
+  rule(:doc_start) { str('<') >> marker >> newline }
+  rule(:marker) { match['A-Z'].repeat(1).capture(:marker) }
+
+  # The content of a document can be either lines of text or another 
+  # document, introduced by <HERE, where HERE is the doc marker. 
+  rule(:text) { (document.as(:doc) | text_line.as(:line)).repeat(1) }
+  rule(:text_line) { captured_marker.absent? >> any >> 
+    (newline.absent? >> any).repeat >> newline }
+    
+  # The end of the document is marked by the marker that was at the beginning
+  # of the document, by itself on a line.
+  rule(:doc_end) { captured_marker }
+  rule(:captured_marker) { 
+    dynamic { |source, context|
+      str(context.captures[:marker])
+    }
+  }
+  
+  rule(:newline) { match["\n"] }
+end
+
+parser = CapturingParser.new
+pp parser.parse_with_debug %Q(<CAPTURE
+Text1
+<FOOBAR
+Text3
+Text4
+FOOBAR
+Text2
+CAPTURE)
+

data/example/ignore.rb

@@ -10,8 +10,8 @@ class IgnoreParslet < Parslet::Atoms::Base
   def to_s_inner(prec)
     @parslet.to_s(prec)
   end
-  def try(source, context)
-    success, value = result = @parslet.try(source, context)
+  def try(source, context, consume_all)
+    success, value = result = @parslet.try(source, context, consume_all)
     
     return succ(nil) if success
     return result

data/example/output/calc.out

@@ -0,0 +1 @@
+123*2 (command line): -> 246

data/example/output/capture.out

@@ -0,0 +1,3 @@
+[{:line=>"Text1\n"@9},
+ {:doc=>[{:line=>"Text3\n"@23}, {:line=>"Text4\n"@29}]},
+ {:line=>"\nText2\n"@41}]

data/example/output/scopes.out

@@ -0,0 +1 @@
+parses 'aba'

data/example/scopes.rb

@@ -0,0 +1,15 @@
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'parslet'
+
+include Parslet
+
+parser = str('a').capture(:a) >> scope { str('b').capture(:a) } >> 
+  dynamic { |s,c| str(c.captures[:a]) }
+  
+begin
+  parser.parse('aba')
+  puts "parses 'aba'"
+rescue
+  puts "exception!"
+end

data/lib/parslet.rb

@@ -83,22 +83,6 @@ module Parslet
     attr_reader :cause 
   end
   
-  # Raised when the parse operation didn't consume all of its input. In this
-  # case, it makes only limited sense to look at the error tree. Maybe the
-  # parser worked just fine, but didn't account for the characters at the tail
-  # of the input?
-  #
-  #   str('foo').parse('foobar')
-  #   # raises Parslet::UnconsumedInput: 
-  #   #   Don't know what to do with "bar" at line 1 char 4.
-  #
-  # Note that you can have parslet ignore this error: 
-  #
-  #   str('foo').parse('foobar', prefix: true)  # => "foo"@0
-  #
-  class UnconsumedInput < ParseFailed
-  end
-  
   module ClassMethods
     # Define an entity for the parser. This generates a method of the same
     # name that can be used as part of other patterns. Those methods can be
@@ -185,6 +169,37 @@ module Parslet
   end
   module_function :any
   
+  # Introduces a new capture scope. This means that all old captures stay
+  # accessible, but new values stored will only be available during the block
+  # given and the old values will be restored after the block. 
+  #
+  # Example: 
+  #   # :a will be available until the end of the block. Afterwards, 
+  #   # :a from the outer scope will be available again, if such a thing 
+  #   # exists. 
+  #   scope { str('a').capture(:a) }
+  #
+  def scope(&block)
+    Parslet::Atoms::Scope.new(block)
+  end
+  module_function :scope
+  
+  # Designates a piece of the parser as being dynamic. Dynamic parsers can
+  # either return a parser at runtime, which will be applied on the input, or
+  # return a result from a parse. 
+  # 
+  # Dynamic parse pieces are never cached and can introduce performance
+  # abnormalitites - use sparingly where other constructs fail. 
+  # 
+  # Example: 
+  #   # Parses either 'a' or 'b', depending on the weather
+  #   dynamic { rand() < 0.5 ? str('a') : str('b') }
+  #   
+  def dynamic(&block)
+    Parslet::Atoms::Dynamic.new(block)
+  end
+  module_function :dynamic
+  
   # A special kind of atom that allows embedding whole treetop expressions
   # into parslet construction. 
   #
@@ -251,4 +266,5 @@ require 'parslet/pattern'
 require 'parslet/pattern/binding'
 require 'parslet/transform'
 require 'parslet/parser'
-require 'parslet/error_reporter'
+require 'parslet/error_reporter'
+require 'parslet/scope'

data/lib/parslet/atoms.rb

@@ -27,5 +27,8 @@ module Parslet::Atoms
   require 'parslet/atoms/re'
   require 'parslet/atoms/str'
   require 'parslet/atoms/entity'
+  require 'parslet/atoms/capture'
+  require 'parslet/atoms/dynamic'
+  require 'parslet/atoms/scope'
 end
 

data/lib/parslet/atoms/alternative.rb

@@ -30,9 +30,9 @@ class Parslet::Atoms::Alternative < Parslet::Atoms::Base
     self.class.new(*@alternatives + [parslet])
   end
   
-  def try(source, context)
+  def try(source, context, consume_all)
     errors = alternatives.map { |a|
-      success, value = result = a.apply(source, context)
+      success, value = result = a.apply(source, context, consume_all)
       return result if success
       
       # Aggregate all errors

data/lib/parslet/atoms/base.rb

@@ -27,7 +27,7 @@ class Parslet::Atoms::Base
 
     # Try to cheat. Assuming that we'll be able to parse the input, don't 
     # run error reporting code. 
-    success, value = setup_and_apply(source, nil)
+    success, value = setup_and_apply(source, nil, !options[:prefix])
     
     # If we didn't succeed the parse, raise an exception for the user. 
     # Stack trace will be off, but the error tree should explain the reason
@@ -36,7 +36,8 @@ class Parslet::Atoms::Base
       # Cheating has not paid off. Now pay the cost: Rerun the parse,
       # gathering error information in the process.
       reporter = options[:reporter] || Parslet::ErrorReporter::Tree.new
-      success, value = setup_and_apply(source, reporter)
+      source.pos = 0
+      success, value = setup_and_apply(source, reporter, !options[:prefix])
       
       fail "Assertion failed: success was true when parsing with reporter" \
         if success
@@ -48,15 +49,12 @@ class Parslet::Atoms::Base
     end
     
     # assert: success is true
-    
-    # If we haven't consumed the input, then the pattern doesn't match. Try
-    # to provide a good error message (even asking down below)
-    if !options[:prefix] && !source.eof?
-      old_pos = source.pos
-      Parslet::Cause.format(
-        source, old_pos, 
-        "Don't know what to do with #{source.consume(10).to_s.inspect}").
-        raise(Parslet::UnconsumedInput)
+
+    # Extra input is now handled inline with the rest of the parsing. If 
+    # really we have success == true, prefix: false and still some input 
+    # is left dangling, that is a BUG.
+    if !options[:prefix] && source.chars_left > 0
+      fail "BUG: New error strategy should not reach this point."
     end
     
     return flatten(value)
@@ -67,21 +65,45 @@ class Parslet::Atoms::Base
   #
   # @return [<Boolean, Object>] Result of the parse. If the first member is 
   #   true, the parse has succeeded. 
-  def setup_and_apply(source, error_reporter)
+  def setup_and_apply(source, error_reporter, consume_all)
     context = Parslet::Atoms::Context.new(error_reporter)
-    apply(source, context)
+    apply(source, context, consume_all)
   end
 
-  #---
-  # Calls the #try method of this parslet. In case of a parse error, apply
-  # leaves the source in the state it was before the attempt. 
-  #+++
-  def apply(source, context)
+  # Calls the #try method of this parslet. Success consumes input, error will 
+  # rewind the input. 
+  #
+  # @param source [Parslet::Source] source to read input from
+  # @param context [Parslet::Atoms::Context] context to use for the parsing
+  # @param consume_all [Boolean] true if the current parse must consume
+  #   all input by itself.
+  def apply(source, context, consume_all=false)
     old_pos = source.pos
     
-    success, value = result = context.try_with_cache(self, source)
+    success, value = result = context.try_with_cache(self, source, consume_all)
 
-    return result if success
+    if success
+      # If a consume_all parse was made and doesn't result in the consumption
+      # of all the input, that is considered an error. 
+      if consume_all && source.chars_left>0
+        # Read 10 characters ahead. Why ten? I don't know. 
+        offending_pos   = source.pos
+        offending_input = source.consume(10)
+        
+        # Rewind input (as happens always in error case)
+        source.pos      = old_pos
+        
+        return context.err_at(
+          self, 
+          source, 
+          "Don't know what to do with #{offending_input.to_s.inspect}", 
+          offending_pos
+        ) 
+      end
+      
+      # Looks like the parse was successful after all. Don't rewind the input.
+      return result
+    end
     
     # We only reach this point if the parse has failed. Rewind the input.
     source.pos = old_pos
@@ -91,11 +113,18 @@ class Parslet::Atoms::Base
   # Override this in your Atoms::Base subclasses to implement parsing
   # behaviour. 
   #
-  def try(source, context)
+  def try(source, context, consume_all)
     raise NotImplementedError, \
       "Atoms::Base doesn't have behaviour, please implement #try(source, context)."
   end
 
+  # Returns true if this atom can be cached in the packrat cache. Most parslet
+  # atoms are cached, so this always returns true, unless overridden.
+  #
+  def cached?
+    true
+  end
+
   # Debug printing - in Treetop syntax. 
   #
   def self.precedence(prec)

data/lib/parslet/atoms/capture.rb

@@ -0,0 +1,38 @@
+
+# Stores the result of matching an atom against input in the #captures in 
+# parse context. Doing so will allow you to pull parts of the ongoing parse
+# out later and use them to match other pieces of input. 
+#
+# Example: 
+#   # After this, context.captures[:an_a] returns 'a'
+#   str('a').capture(:an_a)
+#
+#   # Capture and use of the capture: (matches either 'aa' or 'bb')
+#   match['ab'].capture(:first) >> 
+#     dynamic { |src, ctx| str(ctx.captures[:first]) }
+#   
+class Parslet::Atoms::Capture < Parslet::Atoms::Base
+  attr_reader :parslet, :name
+
+  def initialize(parslet, name)
+    super()
+
+    @parslet, @name = parslet, name
+  end
+
+  def apply(source, context, consume_all)
+    success, value = result = parslet.apply(source, context, consume_all)
+
+    if success
+      context.captures[name.to_sym] = 
+        flatten(value)
+    end
+    
+    return result
+  end
+  
+  def to_s_inner(prec)
+    "(#{name.inspect} = #{parslet.to_s(prec)})"
+  end
+end
+

data/lib/parslet/atoms/context.rb

@@ -12,6 +12,7 @@ module Parslet::Atoms
     def initialize(reporter=Parslet::ErrorReporter::Tree.new)
       @cache = Hash.new { |h, k| h[k] = {} }
       @reporter = reporter
+      @captures = Parslet::Scope.new
     end
     
     # Caches a parse answer for obj at source.pos. Applying the same parslet
@@ -22,14 +23,17 @@ module Parslet::Atoms
     # were consumed by a successful parse. Imitation of such a parse must 
     # advance the input pos by the same amount of bytes.
     #
-    def try_with_cache(obj, source)
+    def try_with_cache(obj, source, consume_all)
       beg = source.pos
         
       # Not in cache yet? Return early.
       unless entry = lookup(obj, beg)
-        result = obj.try(source, self)
+        result = obj.try(source, self, consume_all)
     
-        set obj, beg, [result, source.pos-beg]
+        if obj.cached?
+          set obj, beg, [result, source.pos-beg]
+        end
+        
         return result
       end
 
@@ -59,6 +63,23 @@ module Parslet::Atoms
       return [false, nil]
     end
   
+    # Returns the current captures made on the input (see
+    # Parslet::Atoms::Base#capture). Use as follows: 
+    # 
+    #   context.captures[:foobar] # => returns capture :foobar
+    #
+    attr_reader :captures
+    
+    # Starts a new scope. Use the #scope method of Parslet::Atoms::DSL
+    # to call this. 
+    #
+    def scope
+      captures.push
+      yield
+    ensure
+      captures.pop
+    end
+    
   private 
     def lookup(obj, pos)
       @cache[pos][obj] 

data/lib/parslet/atoms/dsl.rb

@@ -95,4 +95,15 @@ module Parslet::Atoms::DSL
   def as(name)
     Parslet::Atoms::Named.new(self, name)
   end
+
+  # Captures a part of the input and stores it under the name given. This 
+  # is very useful to create self-referential parses. A capture stores
+  # the result of its parse (may be complex) on a successful parse action.
+  # 
+  # Example: 
+  #   str('a').capture(:b)  # will store captures[:b] == 'a'
+  # 
+  def capture(name)
+    Parslet::Atoms::Capture.new(self, name)
+  end
 end

data/lib/parslet/atoms/dynamic.rb

@@ -0,0 +1,32 @@
+# Evaluates a block at parse time. The result from the block must be a parser
+# (something which implements #apply). In the first case, the parser will then
+# be applied to the input, creating the result. 
+#
+# Dynamic parses are never cached. 
+#
+# Example: 
+#   dynamic { rand < 0.5 ? str('a') : str('b') }
+#
+class Parslet::Atoms::Dynamic < Parslet::Atoms::Base
+  attr_reader :block
+  
+  def initialize(block)
+    @block = block
+  end
+  
+  def cached?
+    false
+  end
+  
+  def try(source, context, consume_all)
+    result = block.call(source, context)
+    
+    # Result is a parslet atom.
+    return result.apply(source, context, consume_all)
+  end
+  
+  def to_s_inner(prec)
+    "dynamic { ... }"
+  end
+end
+

data/lib/parslet/atoms/entity.rb

@@ -17,8 +17,8 @@ class Parslet::Atoms::Entity < Parslet::Atoms::Base
     @block = block
   end
 
-  def try(source, context)
-    parslet.apply(source, context)
+  def try(source, context, consume_all)
+    parslet.apply(source, context, consume_all)
   end
   
   def parslet

data/lib/parslet/atoms/lookahead.rb

@@ -21,10 +21,10 @@ class Parslet::Atoms::Lookahead < Parslet::Atoms::Base
     }
   end
   
-  def try(source, context)
+  def try(source, context, consume_all)
     pos = source.pos
 
-    success, value = bound_parslet.apply(source, context)
+    success, value = bound_parslet.apply(source, context, consume_all)
     
     if positive
       return succ(nil) if success

data/lib/parslet/atoms/named.rb

@@ -13,8 +13,8 @@ class Parslet::Atoms::Named < Parslet::Atoms::Base
     @parslet, @name = parslet, name
   end
   
-  def apply(source, context)
-    success, value = result = parslet.apply(source, context)
+  def apply(source, context, consume_all)
+    success, value = result = parslet.apply(source, context, consume_all)
 
     return result unless success
     succ(

data/lib/parslet/atoms/re.rb

@@ -20,12 +20,12 @@ class Parslet::Atoms::Re < Parslet::Atoms::Base
     }
   end
 
-  def try(source, context)
+  def try(source, context, consume_all)
     return succ(source.consume(1)) if source.matches?(re)
     
     # No string could be read
     return context.err(self, source, @error_msgs[:premature]) \
-      if source.eof?
+      if source.chars_left < 1
         
     # No match
     return context.err(self, source, @error_msgs[:failed])

data/lib/parslet/atoms/repetition.rb

@@ -15,18 +15,19 @@ class Parslet::Atoms::Repetition < Parslet::Atoms::Base
     @min, @max = min, max
     @tag = tag
     @error_msgs = {
-      :minrep  => "Expected at least #{min} of #{parslet.inspect}"
+      :minrep  => "Expected at least #{min} of #{parslet.inspect}", 
+      :unconsumed => "Extra input after last repetition"
     }
   end
   
-  def try(source, context)
+  def try(source, context, consume_all)
     occ = 0
     accum = [@tag]   # initialize the result array with the tag (for flattening)
     start_pos = source.pos
     
     break_on = nil
     loop do
-      success, value = parslet.apply(source, context)
+      success, value = parslet.apply(source, context, false)
 
       break_on = value
       break unless success
@@ -49,6 +50,20 @@ class Parslet::Atoms::Repetition < Parslet::Atoms::Base
       start_pos, 
       [break_on]) if occ < min
       
+    # consume_all is true, that means that we're inside the part of the parser
+    # that should consume the input completely. Repetition failing here means
+    # probably that we didn't. 
+    #
+    # We have a special clause to create an error here because otherwise
+    # break_on would get thrown away. It turns out, that contains very
+    # interesting information in a lot of cases. 
+    #
+    return context.err(
+      self, 
+      source, 
+      @error_msgs[:unconsumed], 
+      [break_on]) if consume_all && source.chars_left>0
+      
     return succ(accum)
   end
   

data/lib/parslet/atoms/scope.rb

@@ -0,0 +1,26 @@
+# Starts a new scope in the parsing process. Please also see the #captures
+# method. 
+#
+class Parslet::Atoms::Scope < Parslet::Atoms::Base
+  attr_reader :block
+  def initialize(block)
+    super()
+
+    @block = block
+  end
+  
+  def cached?
+    false
+  end
+  
+  def apply(source, context, consume_all)
+    context.scope do
+      parslet = block.call
+      return parslet.apply(source, context, consume_all)
+    end
+  end
+  
+  def to_s_inner(prec)
+    "scope { #{block.call.to_s(prec)} }"
+  end
+end

data/lib/parslet/atoms/sequence.rb

@@ -19,16 +19,23 @@ class Parslet::Atoms::Sequence < Parslet::Atoms::Base
     self.class.new(* @parslets+[parslet])
   end
   
-  def try(source, context)
-    succ([:sequence]+parslets.map { |p| 
-      success, value = p.apply(source, context) 
+  def try(source, context, consume_all)
+    # Presize an array
+    result = Array.new(parslets.size + 1)
+    result[0] = :sequence
+    
+    parslets.each_with_index do |p, idx|
+      child_consume_all = consume_all && (idx == parslets.size-1)
+      success, value = p.apply(source, context, child_consume_all) 
 
       unless success
         return context.err(self, source, @error_msgs[:failed], [value]) 
       end
       
-      value
-    })
+      result[idx+1] = value
+    end
+    
+    return succ(result)
   end
       
   precedence SEQUENCE

data/lib/parslet/atoms/str.rb

@@ -17,13 +17,14 @@ class Parslet::Atoms::Str < Parslet::Atoms::Base
     }
   end
   
-  def try(source, context)
+  def try(source, context, consume_all)
     return succ(source.consume(@len)) if source.matches?(str)
     
-    # Failures: 
+    # Input ending early:
     return context.err(self, source, @error_msgs[:premature]) \
       if source.chars_left<@len
-      
+    
+    # Expected something, but got something else instead:  
     error_pos = source.pos  
     return context.err_at(
       self, source, 

data/lib/parslet/convenience.rb

@@ -26,8 +26,6 @@ class Parslet::Atoms::Base
   #
   def parse_with_debug str, opts={}
     parse str, opts
-  rescue Parslet::UnconsumedInput => error
-    puts error
   rescue Parslet::ParseFailed => error
     puts error.cause.ascii_tree
   end

data/lib/parslet/parser.rb

@@ -57,8 +57,8 @@ class Parslet::Parser < Parslet::Atoms::Base
     end
   end
   
-  def try(source, context)
-    root.try(source, context)
+  def try(source, context, consume_all)
+    root.try(source, context, consume_all)
   end
   
   def to_s_inner(prec)

data/lib/parslet/scope.rb

@@ -0,0 +1,42 @@
+class Parslet::Scope
+  # Raised when the accessed slot has never been assigned a value. 
+  #
+  class NotFound < StandardError
+  end
+  
+  class Binding
+    attr_reader :parent
+    
+    def initialize(parent=nil)
+      @parent = parent
+      @hash = Hash.new
+    end
+    
+    def [](k)
+      @hash.has_key?(k) && @hash[k] ||
+        parent && parent[k] or 
+        raise NotFound
+    end
+    def []=(k,v)
+      @hash.store(k,v)
+    end
+  end
+  
+  def [](k)
+    @current[k]
+  end
+  def []=(k,v)
+    @current[k] = v
+  end
+  
+  def initialize
+    @current = Binding.new
+  end
+  
+  def push
+    @current = Binding.new(@current)
+  end
+  def pop
+    @current = @current.parent
+  end
+end

data/lib/parslet/source.rb

@@ -47,10 +47,6 @@ module Parslet
       @str.size - @pos
     end
     
-    def eof?
-      @pos >= @str.size
-    end
-
     # Position of the parse as a character offset into the original string. 
     # @note: Encodings...
     attr_accessor :pos

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parslet
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,136 +9,152 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-25 00:00:00.000000000 Z
+date: 2012-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: blankslate
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
+    none: false
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: rspec
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: flexmock
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: rdoc
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: sdoc
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: guard
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: guard-rspec
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: growl
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 description: 
 email: kaspar.schiess@absurd.li
 executables: []
@@ -153,13 +169,16 @@ files:
 - lib/parslet/atoms/alternative.rb
 - lib/parslet/atoms/base.rb
 - lib/parslet/atoms/can_flatten.rb
+- lib/parslet/atoms/capture.rb
 - lib/parslet/atoms/context.rb
 - lib/parslet/atoms/dsl.rb
+- lib/parslet/atoms/dynamic.rb
 - lib/parslet/atoms/entity.rb
 - lib/parslet/atoms/lookahead.rb
 - lib/parslet/atoms/named.rb
 - lib/parslet/atoms/re.rb
 - lib/parslet/atoms/repetition.rb
+- lib/parslet/atoms/scope.rb
 - lib/parslet/atoms/sequence.rb
 - lib/parslet/atoms/str.rb
 - lib/parslet/atoms/visitor.rb
@@ -176,6 +195,7 @@ files:
 - lib/parslet/pattern/binding.rb
 - lib/parslet/pattern.rb
 - lib/parslet/rig/rspec.rb
+- lib/parslet/scope.rb
 - lib/parslet/slice.rb
 - lib/parslet/source/line_cache.rb
 - lib/parslet/source.rb
@@ -183,6 +203,8 @@ files:
 - lib/parslet/transform.rb
 - lib/parslet.rb
 - example/boolean_algebra.rb
+- example/calc.rb
+- example/capture.rb
 - example/comments.rb
 - example/deepest_errors.rb
 - example/documentation.rb
@@ -198,6 +220,8 @@ files:
 - example/modularity.rb
 - example/nested_errors.rb
 - example/output/boolean_algebra.out
+- example/output/calc.out
+- example/output/capture.out
 - example/output/comments.out
 - example/output/deepest_errors.out
 - example/output/documentation.err
@@ -216,12 +240,14 @@ files:
 - example/output/nested_errors.out
 - example/output/parens.out
 - example/output/readme.out
+- example/output/scopes.out
 - example/output/seasons.out
 - example/output/sentence.out
 - example/output/simple_xml.out
 - example/output/string_parser.out
 - example/parens.rb
 - example/readme.rb
+- example/scopes.rb
 - example/seasons.rb
 - example/sentence.rb
 - example/simple.lit
@@ -237,20 +263,17 @@ rdoc_options:
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 1524575203779308108
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -258,3 +281,4 @@ signing_key:
 specification_version: 3
 summary: Parser construction library with great error reporting in Ruby.
 test_files: []
+has_rdoc: