parslet 1.2.3 → 1.3.0

data/HISTORY.txt

@@ -3,6 +3,27 @@
   - prsnt? and absnt? are now finally banned into oblivion. Wasting vocals for
     the win. 
     
+= 1.3.1 / ???
+
+= 1.3.0 / 5Mar2012
+
+  ! Parslet::Transform::Context is now much more well-behaved. It has
+    #respond_to? and #method_missing; it now looks like a plain old Ruby
+    object with instance variables and attribute readers.
+
+  - Grammar transforms turned out to be a dead end and have been removed. 
+
+  ! A few problems in error message generation have been fixed. This will
+  	improve diagnostics further.
+	
+  + A VM driven parser engine: Removes the limitation that parsing needs a 
+    lot of stack space, something dearly missing from Ruby 1.9.3 fibers.
+    This engine is experimental and might be removed in the future. 
+
+  ! Interaction with mathn fixed - Line number generation will terminate. 
+	
+  . Internal reorganisation, removing cruft and bit rot.
+    
 = 1.2.3 / 22Sep2011
 
   + Transform#apply can now be called with a hash as second argument. This 

data/README

@@ -53,6 +53,6 @@ ruby-1.8.7-p334 for better results.
 
 STATUS 
 
-At version 1.2.3 - See HISTORY.txt for changes.
+At version 1.3.0 - See HISTORY.txt for changes.
 
 (c) 2010 Kaspar Schiess

data/example/ignore_whitespace.rb

@@ -0,0 +1,66 @@
+# An example on how to ignore whitespace. Use the composition, luke.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'pp'
+require 'parslet'
+require 'parslet/convenience'
+
+class AParser < Parslet::Parser
+  root :as
+  
+  rule(:as) { a.repeat }
+  rule(:a) { str('a').as(:a) }
+end
+
+class WsIgnoreSource
+  def initialize(string)
+    @io = StringIO.new(string)
+    @early_eof = nil
+  end
+  
+  def pos
+    @io.pos
+  end
+  
+  def pos=(n)
+    @io.pos = n
+  end
+  
+  def gets(buf, n)
+    return nil if eof?
+    
+    return read(n).tap {
+      @early_eof = pos unless can_read?
+    }
+  end
+  
+  def eof?
+    @io.eof? ||                         # the underlying source is EOF
+    @early_eof && pos >= @early_eof     # we have no non-ws chars left
+  end
+  
+  private
+  
+  # Reads n chars from @io.
+  def read(n)
+    b = ''
+    while b.size < n && !@io.eof?
+      c = @io.gets(nil, 1)
+      b << c unless c == ' '
+    end
+    b
+  end
+  
+  # True if there are any chars left in @io.
+  def can_read?
+    old_pos = @io.pos
+    read(1).size == 1
+  rescue => ex
+    return false
+  ensure
+    @io.pos = old_pos
+  end
+end
+
+pp AParser.new.parse_with_debug(WsIgnoreSource.new('a   a a a    '))

data/example/mathn.rb

@@ -0,0 +1,44 @@
+# Demonstrates that we have a compatibility fix to mathn's weird idea of 
+# integer mathematics. 
+# This was contributed by Jonathan Hinkle (https://github.com/hynkle). Thanks!
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parslet'
+require 'parslet/convenience'
+include Parslet
+
+def attempt_parse
+  possible_whitespace = match['\s'].repeat
+
+  cephalopod =
+    str('octopus') |
+    str('squid')
+
+  parenthesized_cephalopod =
+    str('(') >>
+    possible_whitespace >>
+    cephalopod >>
+    possible_whitespace >>
+    str(')')
+
+  parser =
+    possible_whitespace >>
+    parenthesized_cephalopod >>
+    possible_whitespace
+
+  # This parse fails, but that is not the point. When mathn is in the current
+  # ruby environment, it modifies integer division in a way that makes 
+  # parslet loop indefinitely.
+  parser.parse %{(\nsqeed)\n}
+rescue Parslet::ParseFailed
+end
+
+attempt_parse 
+puts 'it terminates before we require mathn'
+
+puts "requiring mathn now"
+require 'mathn'
+puts "and trying again (will hang without the fix)"
+attempt_parse # but it doesn't terminate after requiring mathn
+puts "okay!"

data/example/output/ignore_whitespace.out

@@ -0,0 +1 @@
+[{:a=>"a"@0}, {:a=>"a"@1}, {:a=>"a"@5}, {:a=>"a"@7}]

data/example/output/ip_address.out

@@ -1,9 +1,9 @@
                        0.0.0.0 -> {:ipv4=>"0.0.0.0"@0}
                255.255.255.255 -> {:ipv4=>"255.255.255.255"@0}
-                   255.255.255 -> Failed: Expected one of [IPV4, IPV6]. at line 1 char 1.
+                   255.255.255 -> Failed: Expected one of [IPV4, IPV6] at line 1 char 1.
                1:2:3:4:5:6:7:8 -> {:ipv6=>"1:2:3:4:5:6:7:8"@0}
          12AD:34FC:A453:1922:: -> {:ipv6=>"12AD:34FC:A453:1922::"@0}
                     12AD::34FC -> {:ipv6=>"12AD::34FC"@0}
                         12AD:: -> {:ipv6=>"12AD::"@0}
                             :: -> {:ipv6=>"::"@0}
-                           1:2 -> Failed: Expected one of [IPV4, IPV6]. at line 1 char 1.
+                           1:2 -> Failed: Expected one of [IPV4, IPV6] at line 1 char 1.

data/example/output/mathn.out

@@ -0,0 +1,4 @@
+it terminates before we require mathn
+requiring mathn now
+and trying again (will hang without the fix)
+okay!

data/lib/parslet.rb

@@ -73,6 +73,11 @@ module Parslet
   #   parslet.parse_with_debug(str)
   #
   class ParseFailed < StandardError
+    def initialize(message, cause=nil)
+      super(message)
+      @cause = cause
+    end
+    attr_reader :cause
   end
   
   # Raised when the parse operation didn't consume all of its input. In this
@@ -224,10 +229,12 @@ module Parslet
 end
 
 require 'parslet/slice'
+require 'parslet/cause'
 require 'parslet/source'
 require 'parslet/error_tree'
 require 'parslet/atoms'
 require 'parslet/pattern'
 require 'parslet/pattern/binding'
 require 'parslet/transform'
-require 'parslet/parser'
+require 'parslet/parser'
+require 'parslet/bytecode'

data/lib/parslet/atoms.rb

@@ -15,6 +15,7 @@ module Parslet::Atoms
     OUTER      = (prec+=1)    # printing is done here.
   end
   
+  require 'parslet/atoms/can_flatten'
   require 'parslet/atoms/context'
   require 'parslet/atoms/dsl'
   require 'parslet/atoms/base'

data/lib/parslet/atoms/alternative.rb

@@ -19,7 +19,7 @@ class Parslet::Atoms::Alternative < Parslet::Atoms::Base
     super()
     
     @alternatives = alternatives
-    @error_msg = "Expected one of #{alternatives.inspect}."
+    @error_msg = "Expected one of #{alternatives.inspect}"
   end
 
   #---

data/lib/parslet/atoms/base.rb

@@ -6,6 +6,7 @@
 class Parslet::Atoms::Base
   include Parslet::Atoms::Precedence
   include Parslet::Atoms::DSL
+  include Parslet::Atoms::CanFlatten
   
   # Internally, all parsing functions return either an instance of Fail 
   # or an instance of Success. 
@@ -25,7 +26,23 @@ class Parslet::Atoms::Base
   # and return a result. If the parse fails, a Parslet::ParseFailed exception
   # will be thrown. 
   #
-  def parse(io)
+  def parse(io, traditional=true)
+    if traditional
+      parse_traditional(io)
+    else
+      parse_vm(io)
+    end
+  end
+  
+  def parse_vm(io)
+    compiler = Parslet::Bytecode::Compiler.new
+    program = compiler.compile(self)
+    
+    vm = Parslet::Bytecode::VM.new
+    vm.run(program, io)
+  end
+  
+  def parse_traditional(io)
     source = Parslet::Source.new(io)
     context = Parslet::Atoms::Context.new
     
@@ -36,7 +53,8 @@ class Parslet::Atoms::Base
     # Stack trace will be off, but the error tree should explain the reason
     # it failed.
     if value.error?
-      parse_failed(value.message)
+      @last_cause = value.message
+      @last_cause.raise
     end
     
     # assert: value is a success answer
@@ -48,16 +66,15 @@ class Parslet::Atoms::Base
       # error to fail with. Otherwise just report that we cannot consume the
       # input.
       if cause 
-        # We're not using #parse_failed here, since it assigns to @last_cause.
-        # Still: We'll raise this differently, since the real cause is different.
+        # NOTE We don't overwrite last_cause here.
         raise Parslet::UnconsumedInput, 
           "Unconsumed input, maybe because of this: #{cause}"
       else
         old_pos = source.pos
-        parse_failed(
-          format_cause(source, 
-            "Don't know what to do with #{source.read(100)}", old_pos), 
-          Parslet::UnconsumedInput)
+        @last_cause = source.error(
+          "Don't know what to do with #{source.read(100)}", old_pos)
+
+        @last_cause.raise(Parslet::UnconsumedInput)
       end
     end
     
@@ -94,110 +111,6 @@ class Parslet::Atoms::Base
       "Atoms::Base doesn't have behaviour, please implement #try(source, context)."
   end
 
-  # Takes a mixed value coming out of a parslet and converts it to a return
-  # value for the user by dropping things and merging hashes. 
-  #
-  # Named is set to true if this result will be embedded in a Hash result from 
-  # naming something using <code>.as(...)</code>. It changes the folding 
-  # semantics of repetition.
-  #
-  def flatten(value, named=false) # :nodoc:
-    # Passes through everything that isn't an array of things
-    return value unless value.instance_of? Array
-
-    # Extracts the s-expression tag
-    tag, *tail = value
-
-    # Merges arrays:
-    result = tail.
-      map { |e| flatten(e) }            # first flatten each element
-      
-    case tag
-      when :sequence
-        return flatten_sequence(result)
-      when :maybe
-        return named ? result.first : result.first || ''
-      when :repetition
-        return flatten_repetition(result, named)
-    end
-    
-    fail "BUG: Unknown tag #{tag.inspect}."
-  end
-
-  # Lisp style fold left where the first element builds the basis for 
-  # an inject. 
-  #
-  def foldl(list, &block)
-    return '' if list.empty?
-    list[1..-1].inject(list.first, &block)
-  end
-  
-  # Flatten results from a sequence of parslets. 
-  #
-  def flatten_sequence(list) # :nodoc:
-    foldl(list.compact) { |r, e|        # and then merge flat elements
-      merge_fold(r, e)
-    }
-  end
-  def merge_fold(l, r) # :nodoc:
-    # equal pairs: merge. ----------------------------------------------------
-    if l.class == r.class
-      if l.is_a?(Hash)
-        warn_about_duplicate_keys(l, r)
-        return l.merge(r)
-      else
-        return l + r
-      end
-    end
-    
-    # unequal pairs: hoist to same level. ------------------------------------
-    
-    # Maybe classes are not equal, but both are stringlike?
-    if l.respond_to?(:to_str) && r.respond_to?(:to_str)
-      # if we're merging a String with a Slice, the slice wins. 
-      return r if r.respond_to? :to_slice
-      return l if l.respond_to? :to_slice
-      
-      fail "NOTREACHED: What other stringlike classes are there?"
-    end
-    
-    # special case: If one of them is a string/slice, the other is more important 
-    return l if r.respond_to? :to_str
-    return r if l.respond_to? :to_str
-    
-    # otherwise just create an array for one of them to live in 
-    return l + [r] if r.class == Hash
-    return [l] + r if l.class == Hash
-    
-    fail "Unhandled case when foldr'ing sequence."
-  end
-
-  # Flatten results from a repetition of a single parslet. named indicates
-  # whether the user has named the result or not. If the user has named
-  # the results, we want to leave an empty list alone - otherwise it is 
-  # turned into an empty string. 
-  #
-  def flatten_repetition(list, named) # :nodoc:
-    if list.any? { |e| e.instance_of?(Hash) }
-      # If keyed subtrees are in the array, we'll want to discard all 
-      # strings inbetween. To keep them, name them. 
-      return list.select { |e| e.instance_of?(Hash) }
-    end
-
-    if list.any? { |e| e.instance_of?(Array) }
-      # If any arrays are nested in this array, flatten all arrays to this
-      # level. 
-      return list.
-        select { |e| e.instance_of?(Array) }.
-        flatten(1)
-    end
-    
-    # Consistent handling of empty lists, when we act on a named result        
-    return [] if named && list.empty?
-            
-    # If there are only strings, concatenate them and return that. 
-    foldl(list) { |s,e| s+e }
-  end
 
   # Debug printing - in Treetop syntax. 
   #
@@ -245,51 +158,7 @@ private
   # Produces an instance of Fail and returns it. 
   #
   def error(source, str, pos=nil)
-    @last_cause = format_cause(source, str, pos)
+    @last_cause = source.error(str, pos)
     Fail.new(@last_cause)
   end
-
-  # Signals to the outside that the parse has failed. Use this in conjunction
-  # with #format_cause for nice error messages. 
-  #
-  def parse_failed(cause, exception_klass=Parslet::ParseFailed)
-    @last_cause = cause
-    raise exception_klass,
-      @last_cause.to_s
-  end
-  
-  # An internal class that allows delaying the construction of error messages
-  # (as strings) until we really need to print them. 
-  #
-  class Cause < Struct.new(:message, :source, :pos)
-    def to_s
-      line, column = source.line_and_column(pos)
-      # Allow message to be a list of objects. Join them here, since we now
-      # really need it. 
-      Array(message).map { |o| 
-        o.respond_to?(:to_slice) ? 
-          o.str.inspect : 
-          o.to_s }.join + " at line #{line} char #{column}."
-    end
-  end
-
-  # Appends 'at line ... char ...' to the string given. Use +pos+ to override
-  # the position of the +source+. This method returns an object that can 
-  # be turned into a string using #to_s.
-  #
-  def format_cause(source, str, pos=nil)
-    real_pos = (pos||source.pos)
-    Cause.new(str, source, real_pos)
-  end
-
-  # That annoying warning 'Duplicate subtrees while merging result' comes 
-  # from here. You should add more '.as(...)' names to your intermediary tree.
-  #
-  def warn_about_duplicate_keys(h1, h2)
-    d = h1.keys & h2.keys
-    unless d.empty?
-      warn "Duplicate subtrees while merging result of \n  #{self.inspect}\nonly the values"+
-           " of the latter will be kept. (keys: #{d.inspect})"
-    end
-  end
 end

data/lib/parslet/atoms/can_flatten.rb

@@ -0,0 +1,132 @@
+
+module Parslet::Atoms
+  # A series of helper functions that have the common topic of flattening 
+  # result values into the intermediary tree that consists of Ruby Hashes and 
+  # Arrays. 
+  #
+  # This module has one main function, #flatten, that takes an annotated 
+  # structure as input and returns the reduced form that users expect from 
+  # Atom#parse. 
+  #
+  # NOTE: Since all of these functions are just that, functions without 
+  # side effects, they are in a module and not in a class. Its hard to draw 
+  # the line sometimes, but this is beyond. 
+  #
+  module CanFlatten
+    # Takes a mixed value coming out of a parslet and converts it to a return
+    # value for the user by dropping things and merging hashes. 
+    #
+    # Named is set to true if this result will be embedded in a Hash result from 
+    # naming something using <code>.as(...)</code>. It changes the folding 
+    # semantics of repetition.
+    #
+    def flatten(value, named=false) # :nodoc:
+      # Passes through everything that isn't an array of things
+      return value unless value.instance_of? Array
+
+      # Extracts the s-expression tag
+      tag, *tail = value
+
+      # Merges arrays:
+      result = tail.
+        map { |e| flatten(e) }            # first flatten each element
+
+      case tag
+        when :sequence
+          return flatten_sequence(result)
+        when :maybe
+          return named ? result.first : result.first || ''
+        when :repetition
+          return flatten_repetition(result, named)
+      end
+
+      fail "BUG: Unknown tag #{tag.inspect}."
+    end
+
+    # Lisp style fold left where the first element builds the basis for 
+    # an inject. 
+    #
+    def foldl(list, &block)
+      return '' if list.empty?
+      list[1..-1].inject(list.first, &block)
+    end
+
+    # Flatten results from a sequence of parslets. 
+    #
+    def flatten_sequence(list) # :nodoc:
+      foldl(list.compact) { |r, e|        # and then merge flat elements
+        merge_fold(r, e)
+      }
+    end
+    def merge_fold(l, r) # :nodoc:
+      # equal pairs: merge. ----------------------------------------------------
+      if l.class == r.class
+        if l.is_a?(Hash)
+          warn_about_duplicate_keys(l, r)
+          return l.merge(r)
+        else
+          return l + r
+        end
+      end
+
+      # unequal pairs: hoist to same level. ------------------------------------
+
+      # Maybe classes are not equal, but both are stringlike?
+      if l.respond_to?(:to_str) && r.respond_to?(:to_str)
+        # if we're merging a String with a Slice, the slice wins. 
+        return r if r.respond_to? :to_slice
+        return l if l.respond_to? :to_slice
+
+        fail "NOTREACHED: What other stringlike classes are there?"
+      end
+
+      # special case: If one of them is a string/slice, the other is more important 
+      return l if r.respond_to? :to_str
+      return r if l.respond_to? :to_str
+
+      # otherwise just create an array for one of them to live in 
+      return l + [r] if r.class == Hash
+      return [l] + r if l.class == Hash
+
+      fail "Unhandled case when foldr'ing sequence."
+    end
+
+    # Flatten results from a repetition of a single parslet. named indicates
+    # whether the user has named the result or not. If the user has named
+    # the results, we want to leave an empty list alone - otherwise it is 
+    # turned into an empty string. 
+    #
+    def flatten_repetition(list, named) # :nodoc:
+      if list.any? { |e| e.instance_of?(Hash) }
+        # If keyed subtrees are in the array, we'll want to discard all 
+        # strings inbetween. To keep them, name them. 
+        return list.select { |e| e.instance_of?(Hash) }
+      end
+
+      if list.any? { |e| e.instance_of?(Array) }
+        # If any arrays are nested in this array, flatten all arrays to this
+        # level. 
+        return list.
+          select { |e| e.instance_of?(Array) }.
+          flatten(1)
+      end
+
+      # Consistent handling of empty lists, when we act on a named result        
+      return [] if named && list.empty?
+
+      # If there are only strings, concatenate them and return that. 
+      foldl(list) { |s,e| s+e }
+    end
+
+    # That annoying warning 'Duplicate subtrees while merging result' comes 
+    # from here. You should add more '.as(...)' names to your intermediary tree.
+    #
+    def warn_about_duplicate_keys(h1, h2)
+      d = h1.keys & h2.keys
+      unless d.empty?
+        warn "Duplicate subtrees while merging result of \n  #{self.inspect}\nonly the values"+
+             " of the latter will be kept. (keys: #{d.inspect})"
+      end
+    end
+  end
+end