parslet 1.1.1 → 1.2.0

data/HISTORY.txt

@@ -1,3 +1,32 @@
+= 2.0 / ?? (future release changes, like a reminder to self)
+  
+  - prsnt? and absnt? are now finally banned into oblivion. Wasting vocals for
+    the win. 
+
+= 1.2.0 / ???
+  
+  + Parslet::Parser is now also a grammar atom, it can be composed freely with
+    other atoms. (str('f') >> MiniLispParser.new >> str('b'))
+    
+  + No strings, only slices are returned as part of the parser result. 
+    Parslet::Slice is almost a string class, but one that remembers the 
+    source offset. This has also bought us a slight speedup.
+    
+  + require 'parslet/convenience' now brings #parse_with_debug to all parslets.
+    This is a consequence of the above change. 
+  
+  + Deprecates prsnt? and absnt? in favor of the more readable absent? and
+    prsnt?. Uses 3 bytes more RAM. The old variants will exist until we release
+    2.0. 
+  
+  INTERNALLY
+  
+  + Visitors now should have methods that all begin with 'visit_*'. #str 
+    becomes #visit_str.
+
+  + Parslet::Atoms::Entity now takes only a block argument instead of context
+    and block. 
+
 = 1.1.1 / 4Feb2011
 
   ! FIX: Line counting was broken by performance optimisations. 

data/README

@@ -27,9 +27,7 @@ SYNOPSIS
             str('"')
   
   # Parse the string and capture parts of the interpretation (:string above)        
-  tree = parser.parse(%Q{
-    "This is a \\"String\\" in which you can escape stuff"
-  }.strip)
+  tree = parser.parse('"This is a \\"String\\" in which you can escape stuff"')
 
   tree # => {:string=>"This is a \\\"String\\\" in which you can escape stuff"}
 
@@ -37,7 +35,7 @@ SYNOPSIS
 
   transform = Parslet::Transform.new do
     rule(:string => simple(:x)) { 
-      puts "String contents (method 2): #{x}" }
+      puts "String contents: #{x}" }
   end
   transform.apply(tree)
 

data/Rakefile

@@ -1,13 +1,18 @@
-
 require "rubygems"
 require "rake/rdoctask"
 require 'rspec/core/rake_task'
 require "rake/gempackagetask"
 
-
-desc "Run all examples"
+desc "Run all tests: Exhaustive."
 RSpec::Core::RakeTask.new
 
+namespace :spec do
+  desc "Only run unit tests: Fast. "
+  RSpec::Core::RakeTask.new(:unit) do |task|
+    task.pattern = "spec/parslet/**/*_spec.rb"
+  end
+end
+
 task :default => :spec
 
 require 'sdoc'
@@ -27,10 +32,19 @@ desc 'Clear out RDoc'
 task :clean => [:clobber_rdoc, :clobber_package]
 
 # This task actually builds the gem. 
+task :gem => :spec
 spec = eval(File.read('parslet.gemspec'))
+
 desc "Generate the gem package."
 Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
 end
 
-task :gem => :spec
+desc "Prints LOC stats"
+task :stat do
+  %w(lib spec example).each do |dir|
+    loc = %x(find #{dir} -name "*.rb" | xargs wc -l | grep 'total').split.first.to_i
+    printf("%20s %d\n", dir, loc)
+  end
+end
+

data/example/comments.rb

@@ -1,26 +1,26 @@
 # A small example on how to parse common types of comments. The example
 # started out with parser code from Stephen Waits. 
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'pp'
 require 'parslet'
 require 'parslet/convenience'
 
 class ALanguage < Parslet::Parser
-  root(:expressions)
+  root(:lines)
   
-  rule(:expressions) { (line >> eol).repeat(1) | line }
-  rule(:line) { space? >> an_expression.as(:exp).repeat }
-  rule(:an_expression) { str('a').as(:a) >> space? }
+  rule(:lines) { line.repeat }
+  rule(:line) { spaces >> expression.repeat >> newline }
+  rule(:newline) { str("\n") >> str("\r").maybe }
   
-  rule(:eol) { space? >> match["\n\r"].repeat(1) >> space? }
+  rule(:expression) { (str('a').as(:a) >> spaces).as(:exp) }
   
-  rule(:space?) { space.repeat }
-  rule(:space) { multiline_comment.as(:multi) | line_comment.as(:line) | str(' ') }
-
-  rule(:line_comment) { str('//') >> (match["\n\r"].absnt? >> any).repeat }
-  rule(:multiline_comment) { str('/*') >> (str('*/').absnt? >> any).repeat >> str('*/') }
+  rule(:spaces) { space.repeat }
+  rule(:space) { multiline_comment | line_comment | str(' ') }
+  
+  rule(:line_comment) { (str('//') >> (newline.absent? >> any).repeat).as(:line) }
+  rule(:multiline_comment) { (str('/*') >> (str('*/').absent? >> any).repeat >> str('*/')).as(:multi) }
 end
 
 code = %q(
@@ -33,5 +33,3 @@ code = %q(
 )
 
 pp ALanguage.new.parse_with_debug(code)
-
-

data/example/documentation.rb

@@ -1,7 +1,7 @@
 # A small example that shows a really small parser and what happens on parser
 # errors. 
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'pp'
 require 'parslet'

data/example/email_parser.rb

@@ -2,7 +2,7 @@
 
 # Example contributed by Hal Brodigan (postmodern). Thanks!
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 require 'parslet'
 
 class EmailParser < Parslet::Parser
@@ -35,7 +35,7 @@ class EmailSanitizer < Parslet::Transform
   rule(:word => simple(:word)) { word }
 
   rule(:username => sequence(:username)) { username.join + "@" }
-  rule(:username => simple(:username)) { username + "@" }
+  rule(:username => simple(:username)) { username.to_s + "@" }
 
   rule(:email => sequence(:email)) { email.join }
 end
@@ -45,12 +45,12 @@ sanitizer = EmailSanitizer.new
 
 unless ARGV[0]
   STDERR.puts "usage: #{$0} \"EMAIL_ADDR\""
-  exit -1
+  STDOUT.puts "since you haven't specified any EMAIL_ADDR, for testing purposes we're using a.b.c.d@gmail.com"
 end
 
 begin
-  p sanitizer.apply(parser.parse(ARGV[0]))
+  p sanitizer.apply(parser.parse(ARGV[0] || 'a.b.c.d@gmail.com'))
 rescue Parslet::ParseFailed => error
   puts error
-  puts parser.root.error_tree
+  puts parser.error_tree
 end

data/example/empty.rb

@@ -1,7 +1,7 @@
 # Basically just demonstrates that you can leave rules empty and get a nice
 # NotImplementedError. A way to quickly spec out your parser rules?
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'parslet'
 
@@ -10,4 +10,4 @@ class Parser < Parslet::Parser
 end
 
 
-Parser.new.empty.parslet
+Parser.new.empty.parslet

data/example/erb.rb

@@ -1,8 +1,11 @@
+# Example that demonstrates how a simple erb-like parser could be constructed. 
+
 $:.unshift File.dirname(__FILE__) + "/../lib"
+
 require 'parslet'
 
 class ErbParser < Parslet::Parser
-  rule(:ruby) { (str('%>').absnt? >> any).repeat.as(:ruby) }
+  rule(:ruby) { (str('%>').absent? >> any).repeat.as(:ruby) }
   
   rule(:expression) { (str('=') >> ruby).as(:expression) }
   rule(:comment) { (str('#') >> ruby).as(:comment) }
@@ -10,7 +13,7 @@ class ErbParser < Parslet::Parser
   rule(:erb) { expression | comment | code }
   
   rule(:erb_with_tags) { str('<%') >> erb >> str('%>') }
-  rule(:text) { (str('<%').absnt? >> any).repeat(1) }
+  rule(:text) { (str('<%').absent? >> any).repeat(1) }
   
   rule(:text_with_ruby) { (text.as(:text) | erb_with_tags).repeat.as(:text) }
   root(:text_with_ruby)
@@ -41,4 +44,4 @@ The <%# a = 1 %>not printed non-evaluated comment "a = 1", see the value of a be
 The <%= 'nicely' %> printed result.
 The <% b = 3 %>value of a is <%= a %>, and b is <%= b %>.
 ERB
-))
+))

data/example/ip_address.rb

@@ -6,7 +6,7 @@
 #
 # See http://tools.ietf.org/html/rfc3986#appendix-A for more information.
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'pp'
 require 'parslet'
@@ -122,4 +122,4 @@ end
   rescue Parslet::ParseFailed => m
     puts "Failed: #{m}"
   end
-end
+end

data/example/local.rb

@@ -0,0 +1,34 @@
+
+# An exploration of two ideas: 
+#   a) Constructing a whole parser inline, without the artificial class around
+#      it. 
+# and:
+#   b) Constructing non-greedy or non-blind parsers by transforming the 
+#      grammar.
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+
+require 'parslet'
+include Parslet
+
+a =  str('a').repeat >> str('aa')
+
+# E1% E2
+# 
+# S = E2 | E1 S 
+
+def this(name, &block); return Parslet::Atoms::Entity.new(name, &block) end
+def epsilon; any.absent? end 
+
+# Traditional repetition will try as long as the pattern can be matched and 
+# then give up. This is greedy and blind. 
+a = str('a').as(:e) >> this('a') { a }.as(:rec) | epsilon
+
+# Here's a pattern match that is greedy and non-blind. The first pattern
+# 'a'* will be tried as many times as possible, while still matching the 
+# end pattern 'aa'.
+b = str('aa').as(:e2) >> epsilon | str('a').as(:e1) >> this('b') { b }.as(:rec)
+
+p a.parse('aaaa')
+p b
+p b.parse('aaaa')

data/example/minilisp.rb

@@ -1,7 +1,7 @@
 # Reproduces [1] using parslet. 
 # [1] http://thingsaaronmade.com/blog/a-quick-intro-to-writing-a-parser-using-treetop.html
 
-$:.unshift '../lib'
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'pp'
 require 'parslet'
@@ -45,7 +45,7 @@ module MiniLisp
     rule(:string) {
       str('"') >> (
         str('\\') >> any |
-        str('"').absnt? >> any 
+        str('"').absent? >> any 
       ).repeat.as(:string) >> str('"') >> space?
     }
   end

data/example/output/comments.out

@@ -0,0 +1,8 @@
+[{:exp=>{:a=>"a"@3}},
+ {:line=>"// line comment"@7},
+ {:exp=>{:a=>"a"@25}},
+ {:exp=>{:a=>"a"@27}},
+ {:exp=>[{:a=>"a"@29}, {:line=>"// line comment"@31}]},
+ {:exp=>[{:a=>"a"@49}, {:multi=>"/* inline comment */"@51}]},
+ {:exp=>{:a=>"a"@72}},
+ {:multi=>"/* multiline\n  comment */"@77}]

data/example/output/documentation.err

@@ -0,0 +1,4 @@
+/Users/kaspar/git_work/own/parslet/lib/parslet/atoms/base.rb:326:in `parse_failed': Don't know what to do with bbbb at line 1 char 1. (Parslet::ParseFailed)
+	from /Users/kaspar/git_work/own/parslet/lib/parslet/atoms/base.rb:55:in `parse'
+	from example/documentation.rb:13:in `parse'
+	from example/documentation.rb:18:in `<main>'

data/example/output/documentation.out

@@ -0,0 +1 @@
+"aaaa"@0

data/example/output/email_parser.out

@@ -0,0 +1,2 @@
+since you haven't specified any EMAIL_ADDR, for testing purposes we're using a.b.c.d@gmail.com
+"a.b.c.d@gmail.com"

data/example/output/empty.err

@@ -0,0 +1 @@
+example/empty.rb:13:in `<main>': rule(:empty) { ... }  returns nil. Still not implemented, but already used? (NotImplementedError)

data/example/output/erb.out

@@ -0,0 +1,7 @@
+{:text=>[{:text=>"The value of x is "@0}, {:expression=>{:ruby=>" x "@21}}, {:text=>"."@26}]}
+{:text=>[{:code=>{:ruby=>" 1 + 2 "@2}}]}
+{:text=>[{:comment=>{:ruby=>" commented "@3}}]}
+The not printed result of "a = 2".
+The not printed non-evaluated comment "a = 1", see the value of a below.
+The nicely printed result.
+The value of a is 2, and b is 3.

data/example/output/ip_address.out

@@ -0,0 +1,9 @@
+                       0.0.0.0 -> {:ipv4=>"0.0.0.0"@0}
+               255.255.255.255 -> {:ipv4=>"255.255.255.255"@0}
+                   255.255.255 -> Failed: Expected one of [IPV4, IPV6]. at line 1 char 1.
+               1:2:3:4:5:6:7:8 -> {:ipv6=>"1:2:3:4:5:6:7:8"@0}
+         12AD:34FC:A453:1922:: -> {:ipv6=>"12AD:34FC:A453:1922::"@0}
+                    12AD::34FC -> {:ipv6=>"12AD::34FC"@0}
+                        12AD:: -> {:ipv6=>"12AD::"@0}
+                            :: -> {:ipv6=>"::"@0}
+                           1:2 -> Failed: Expected one of [IPV4, IPV6]. at line 1 char 1.

data/example/output/local.out

@@ -0,0 +1,3 @@
+{:e=>"a"@0, :rec=>{:e=>"a"@1, :rec=>{:e=>"a"@2, :rec=>{:e=>"a"@3, :rec=>nil}}}}
+e2:'aa' !. / e1:'a' rec:B
+{:e1=>"a"@0, :rec=>{:e1=>"a"@1, :rec=>{:e2=>"aa"@2}}}

data/example/output/minilisp.out

@@ -0,0 +1,5 @@
+[:define,
+ :test,
+ [:lambda,
+  [],
+  [:begin, [:display, "something"@54], [:display, 1], [:display, 3.08]]]]

data/example/output/parens.out

@@ -0,0 +1,8 @@
+                  (): {:l=>"("@0, :m=>nil, :r=>")"@1} (1 parens)
+
+                (()): {:l=>"("@0, :m=>{:l=>"("@1, :m=>nil, :r=>")"@2}, :r=>")"@3} (2 parens)
+
+          ((((())))): {:l=>"("@0, :m=>{:l=>"("@1, :m=>{:l=>"("@2, :m=>{:l=>"("@3, :m=>{:l=>"("@4, :m=>nil, :r=>")"@5}, :r=>")"@6}, :r=>")"@7}, :r=>")"@8}, :r=>")"@9} (5 parens)
+
+               ((()): Failed to match sequence (l:'(' m:(BALANCED?)) at line 1 char 6.
+

data/example/output/readme.out

@@ -0,0 +1 @@
+String contents: This is a \"String\" in which you can escape stuff

data/example/output/seasons.out

@@ -0,0 +1,28 @@
+"And when Spring comes"
+{:bud=>{:stem=>[{:branch=>:leaf}]}}
+
+"And when Summer comes"
+{:bud=>{:stem=>[{:branch=>[:leaf, :flower]}]}}
+
+"And when Fall comes"
+Fruit!
+Falling Leaves!
+{:bud=>{:stem=>[{:branch=>[]}]}}
+
+"And when Winter comes"
+{:bud=>{:stem=>[]}}
+
+"And when Spring comes"
+{:bud=>{:stem=>[{:branch=>:leaf}]}}
+
+"And when Summer comes"
+{:bud=>{:stem=>[{:branch=>[:leaf, :flower]}]}}
+
+"And when Fall comes"
+Fruit!
+Falling Leaves!
+{:bud=>{:stem=>[{:branch=>[]}]}}
+
+"And when Winter comes"
+{:bud=>{:stem=>[]}}
+

data/example/output/simple_xml.out

@@ -0,0 +1,2 @@
+"verified"
+{:o=>{:name=>"b"@1}, :i=>"verified", :c=>{:name=>"a"@33}}

data/example/output/string_parser.out

@@ -0,0 +1,3 @@
+[#<struct IntLit text="123"@0>,
+ #<struct IntLit text="12345"@4>,
+ #<struct StringLit text=" Some String with \\\"escapes\\\""@11>]

data/example/parens.rb

@@ -2,9 +2,7 @@
 # uses '.as(:name)' to construct a tree that can reliably be matched
 # afterwards. 
 
-$:.unshift File.join(
-  File.dirname(__FILE__), 
-  '/../lib')
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
 require 'pp'
 require 'parslet'

data/example/readme.rb

@@ -1,12 +1,9 @@
 # The example from the readme. With this, I am making sure that the readme 
 # 'works'. Is this too messy?
 
-$:.unshift '../lib'
-
-require 'pp'
-require 'parslet'
-include Parslet
+$:.unshift File.dirname(__FILE__) + "/../lib"
 
+# cut here -------------------------------------------------------------------
 require 'parslet'
 include Parslet
 
@@ -14,20 +11,17 @@ include Parslet
 parser =  str('"') >> 
           (
             str('\\') >> any |
-            str('"').absnt? >> any
+            str('"').absent? >> any
           ).repeat.as(:string) >> 
           str('"')
   
 # Parse the string and capture parts of the interpretation (:string above)        
-tree = parser.parse(%Q{
-  "This is a \\"String\\" in which you can escape stuff"
-}.strip)
+tree = parser.parse('"This is a \\"String\\" in which you can escape stuff"')
 
 tree # => {:string=>"This is a \\\"String\\\" in which you can escape stuff"}
 
 # Here's how you can grab results from that tree:
 
-# 1)
 transform = Parslet::Transform.new do
   rule(:string => simple(:x)) { 
     puts "String contents: #{x}" }