ruby_parser 3.0.0.a5 → 3.0.0.a6

data/.autotest

@@ -11,6 +11,7 @@ Autotest.add_hook :initialize do |at|
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
   at.add_exception '.diff'
+  at.add_exception 'rubycorpus'
 
   at.libs << ':../../minitest/dev/lib'
   at.testlib = "minitest/autorun"

data/History.txt

@@ -1,3 +1,21 @@
+=== 3.0.0.a6 / 2012-08-20
+
+* 2 minor enhancements:
+
+  * 1.8: Added basic encoding support to 1.8 parser. Assumes -Ku.
+  * 1.9: Added encoding magic comment support to 1.9 parser.
+
+* 8 bug fixes:
+
+  * 1.9: Fixed lexing of -1 w/in conditionals. yeah... I dunno.
+  * 1.9: Fixed parsing of a do | | end.
+  * 1.9: Fixed parsing of not(x).
+  * 1.9: Fixed parsing of op_asgn + rescue: 'a ||= b rescue nil'
+  * 1.9: added \r to the EOL backslash handler. dos files blow
+  * 1.9: hacked in a workaround for 1.9 specific regexps running in 1.8.
+  * Added #reset to RubyParser proxy class
+  * Fixed lexing of conditional w/ %() tokens
+
 === 3.0.0.a5 / 2012-07-31
 
 * 5 bug fixes:

data/bin/ruby_parse_extract_error

@@ -1,5 +1,10 @@
 #!/usr/bin/ruby -ws
 
+$d ||= false
+$d ||= ENV["DELETE"]
+$q ||= false
+$q ||= ENV["QUIET"]
+
 require 'rubygems'
 require 'ruby_parser'
 
@@ -13,6 +18,8 @@ class Racc::Parser
 
     defs = pre_error.grep(/^ *def/)
 
+    raise "can't figure out where the bad code starts" unless defs.last
+
     last_def_indent = defs.last[/^ */]
 
     post_error = src[ss.pos..-1]
@@ -30,7 +37,9 @@ class Racc::Parser
 
     parser = self.class.new
 
-    parser.process(defs.join("\n\n")) rescue nil
+    parser.process(defs.join("\n\n"))
+  rescue SyntaxError, StandardError
+    nil
   end
 end
 
@@ -46,34 +55,46 @@ ARGV.replace ARGV.map { |path|
   f =~ /bad_ruby_file/ # I have intentionally bad code in my test dirs
 }
 
-parser = Ruby19Parser.new
-
 ARGV.each do |file|
   ruby = file == "-" ? $stdin.read : File.read(file)
 
   begin
-    $stderr.print "Validating #{file}: "
+    $stderr.print "# Validating #{file}: "
+    parser = Ruby19Parser.new
     parser.process(ruby, file)
-    $stderr.puts "good"
-  rescue RuntimeError, Racc::ParseError => e
-    $stderr.puts "error: #{e.message.strip}"
-    $stderr.puts
+    warn "good"
+    File.unlink file if $d
+  rescue StandardError, SyntaxError, Racc::ParseError => e
+    warn "# error: #{e.message.strip}" unless $q
+    warn ""
+    next if $q
 
-    defs = parser.extract_defs
+    begin
+      defs = parser.extract_defs
 
-    abort "Can't reproduce error with just methods, punting..." if
-      parser.retest_for_errors defs
+      orig_size = defs.size
 
-    mandatory = defs.pop
+      if parser.retest_for_errors defs then
+        warn "Can't reproduce error with just methods, punting..."
+        next
+      end
 
-    (1..defs.size).each do |perm_size|
-      defs.combination(perm_size).each do |trial|
-        trial << mandatory
-        unless parser.retest_for_errors trial then
-          puts trial.join "\n"
-          abort "reduced repro found!"
+      mandatory = defs.pop
+
+      catch :extract_done do
+        (1..defs.size).each do |perm_size|
+          defs.combination(perm_size).each do |trial|
+            trial << mandatory
+            unless parser.retest_for_errors trial then
+              puts trial.join "\n"
+              warn "# reduced repro found!"
+              throw :extract_done
+            end
+          end
         end
       end
+    rescue RuntimeError, Racc::ParseError => e
+      warn "# error: #{e.message.strip}"
     end
   end
 end

data/lib/ruby18_parser.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.8
+# This file is automatically generated by Racc 1.4.9
 # from Racc grammer file "".
 #
 

data/lib/ruby19_parser.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.8
+# This file is automatically generated by Racc 1.4.9
 # from Racc grammer file "".
 #
 
@@ -3992,7 +3992,7 @@ end
 
 def _reduce_198(val, _values, result)
                       result = new_op_asgn val
-                      result = self.node_assign s(:rescue, result, new_resbody(s(:array), val[4]))
+                      result = s(:rescue, result, new_resbody(s(:array), val[4]))
                     
     result
 end
@@ -4540,7 +4540,7 @@ def _reduce_292(val, _values, result)
 end
 
 def _reduce_293(val, _values, result)
-                      raise "no2: #{val.inspect}"
+                      result = s(:call, val[2], :"!")
                     
     result
 end
@@ -5029,7 +5029,7 @@ end
 # reduce 383 omitted
 
 def _reduce_384(val, _values, result)
-                      raise "no17: #{val.inspect}"
+                      result = val[1] || 0
                     
     result
 end

data/lib/ruby19_parser.y

@@ -582,7 +582,7 @@ rule
                 | var_lhs tOP_ASGN arg kRESCUE_MOD arg
                     {
                       result = new_op_asgn val
-                      result = self.node_assign s(:rescue, result, new_resbody(s(:array), val[4]))
+                      result = s(:rescue, result, new_resbody(s(:array), val[4]))
                     }
                 | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg
                     {
@@ -966,7 +966,7 @@ rule
                     }
                 | kNOT tLPAREN2 expr rparen
                     {
-                      raise "no2: #{val.inspect}"
+                      result = s(:call, val[2], :"!")
                     }
                 | kNOT tLPAREN2 rparen
                     {
@@ -1309,7 +1309,7 @@ rule
 
  block_param_def: tPIPE opt_bv_decl tPIPE
                     {
-                      raise "no17: #{val.inspect}"
+                      result = val[1] || 0
                     }
                 | tOROP
                     {

data/lib/ruby_lexer.rb

@@ -1,4 +1,18 @@
+# encoding: US-ASCII
+
 class RubyLexer
+
+  IDENT_CHAR_RE = case RUBY_VERSION
+                  when /^1\.8/ then
+                    /[\w\x80-\xFF]/
+                  when /^(1\.9|2\.0)/ then # HACK - matching 2.0 for now
+                    /[\w\u0080-\uFFFF]/u
+                  else
+                    raise "bork"
+                  end
+
+  IDENT_RE = /^#{IDENT_CHAR_RE}+/
+
   attr_accessor :command_start
   attr_accessor :cmdarg
   attr_accessor :cond
@@ -773,7 +787,7 @@ class RubyLexer
           self.lex_state = :expr_dot
           self.yacc_value = "::"
           return :tCOLON2
-        elsif ! is_end? && src.scan(/:([a-zA-Z_]\w*(?:[?!]|=(?!>))?)/) then
+        elsif ! is_end? && src.scan(/:([a-zA-Z_]#{IDENT_CHAR_RE}*(?:[?!]|=(?!>))?)/) then
           # scanning shortcut to symbols
           self.yacc_value = src[1]
           self.lex_state = :expr_end
@@ -898,7 +912,7 @@ class RubyLexer
             return :tOP_ASGN
           end
 
-          if (lex_state == :expr_beg || lex_state == :expr_mid ||
+          if (is_beg? ||
               (lex_state.is_argument && space_seen && !src.check(/\s/))) then
             if lex_state.is_argument then
               arg_ambiguous
@@ -1144,14 +1158,14 @@ class RubyLexer
 
           return :tTILDE
         elsif src.scan(/\\/) then
-          if src.scan(/\n/) then
+          if src.scan(/\r?\n/) then
             self.lineno = nil
             self.space_seen = true
             next
           end
           rb_compile_error "bare backslash only allowed before newline"
         elsif src.scan(/\%/) then
-          if lex_state == :expr_beg || lex_state == :expr_mid then
+          if is_beg? then
             return parse_quote
           end
 
@@ -1229,12 +1243,12 @@ class RubyLexer
       if src.scan(/\004|\032|\000/) || src.eos? then # ^D, ^Z, EOF
         return RubyLexer::EOF
       else # alpha check
-        if src.scan(/\W/) then
+        unless src.check IDENT_RE then
           rb_compile_error "Invalid char #{src.matched.inspect} in expression"
         end
       end
 
-      self.token = src.matched if self.src.scan(/\w+/)
+      self.token = src.matched if self.src.scan IDENT_RE
 
       return process_token(command_state)
     end
@@ -1311,7 +1325,7 @@ class RubyLexer
 
   def process_token(command_state)
 
-    token << src.matched if token =~ /^\w/ && src.scan(/[\!\?](?!=)/)
+    token << src.matched if token =~ IDENT_RE && src.scan(/[\!\?](?!=)/)
 
     result = nil
     last_state = lex_state

data/lib/ruby_parser_extras.rb

@@ -78,7 +78,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a5' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a6' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -749,7 +749,7 @@ module RubyParserStuff
                   Regexp.new(node[1], o, k)
                 else
                   Regexp.new(node[1], o)
-                end
+                end rescue node[1] # HACK - drops options
     when :dstr then
       if options =~ /o/ then
         node[0] = :dregx_once
@@ -902,8 +902,21 @@ module RubyParserStuff
   def process(str, file = "(string)")
     raise "bad val: #{str.inspect}" unless String === str
 
+    str.lines.first(2).find { |s| s[/^# encoding: (.+)/, 1] }
+    encoding = $1
+
+    str = str.dup
+
+    if encoding then
+      if defined?(Encoding) then
+        str.force_encoding(encoding).encode! "utf-8"
+      else
+        warn "Skipping magic encoding comment"
+      end
+    end
+
     self.file = file
-    self.lexer.src = str.dup
+    self.lexer.src = str
 
     @yydebug = ENV.has_key? 'DEBUG'
 
@@ -1199,13 +1212,18 @@ class RubyParser
     @p19 = Ruby19Parser.new
   end
 
-  def process s, f = "(string)"
+  def process(s, f = "(string)") # parens for emacs *sigh*
     Ruby19Parser.new.process s, f
   rescue Racc::ParseError
     Ruby18Parser.new.process s, f
   end
 
   alias :parse :process
+
+  def reset
+    @p18.reset
+    @p19.reset
+  end
 end
 
 ############################################################

data/test/test_ruby_parser.rb

@@ -1,4 +1,5 @@
 #!/usr/local/bin/ruby
+# encoding: utf-8
 
 # ENV['VERBOSE'] = "1"
 
@@ -646,6 +647,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_bug_cond_pct
+    rb = "case; when %r%blahblah%; end"
+    pt = s(:case, nil, s(:when, s(:array, s(:lit, /blahblah/)), nil), nil)
+
+    assert_parse rb, pt
+  end
+
   # according to 2.3.1 parser:
   # rp.process("f { |(a,b),c| }") == rp.process("f { |((a,b),c)| }")
 
@@ -657,7 +665,7 @@ module TestRubyParserShared
   #            s(:array,
   #              s(:masgn, s(:array, s(:lasgn, :a), s(:lasgn, :b))),
   #              s(:lasgn, :c))))
-  # 
+  #
   #   assert_parse rb, pt.dup
   # end
 
@@ -690,11 +698,18 @@ module TestRubyParserShared
   #   assert_parse rb, pt
   # end
 
+  def ruby18
+    Ruby18Parser === self.processor
+  end
+
+  def ruby19
+    Ruby19Parser === self.processor
+  end
+
   def test_bug_comma
-    val = case self.processor
-          when Ruby18Parser then
+    val = if ruby18 then
             s(:lit, 100)
-          when Ruby19Parser then
+          elsif ruby19 then
             s(:str, "d")
           else
             raise "wtf"
@@ -745,6 +760,68 @@ module TestRubyParserShared
 
     assert_parse rb, pt
   end
+
+
+  def test_bug_not_parens
+    rb = "not(a)"
+    pt = if ruby18 then
+           s(:not, s(:call, nil, :a))
+         elsif ruby19 then
+           s(:call, s(:call, nil, :a), :"!")
+         else
+           raise "wtf"
+         end
+
+    assert_parse rb, pt
+  end
+
+  def test_pipe_space
+    rb = "a.b do | | end"
+    pt = s(:iter, s(:call, s(:call, nil, :a), :b), 0)
+
+    assert_parse rb, pt
+  end
+
+  def test_cond_unary_minus
+    rb = "if -1; end"
+    pt = s(:if, s(:lit, -1), nil, nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_bug_op_asgn_rescue
+    rb = "a ||= b rescue nil"
+    pt = s(:rescue,
+           s(:op_asgn_or, s(:lvar, :a), s(:lasgn, :a, s(:call, nil, :b))),
+           s(:resbody, s(:array), s(:nil)))
+
+    assert_parse rb, pt
+  end
+
+  def test_magic_encoding_comment
+    rb = <<-EOM.gsub(/^      /, '')
+      # encoding: utf-8
+      class ExampleUTF8ClassNameVarietà
+        def self.è
+          così = :però
+        end
+      end
+    EOM
+
+    # TODO: class vars
+    # TODO: odd-ternary: a ?bb : c
+    # TODO: globals
+
+    pt = s(:class, :"ExampleUTF8ClassNameVariet\303\240", nil,
+           s(:defs, s(:self), :"\303\250", s(:args),
+             s(:lasgn, :"cos\303\254", s(:lit, :"per\303\262"))))
+
+    err = RUBY_VERSION =~ /^1\.8/ ? "Skipping magic encoding comment\n" : ""
+
+    assert_output "", err do
+      assert_parse rb, pt
+    end
+  end
 end
 
 class TestRubyParser < MiniTest::Unit::TestCase
@@ -1112,6 +1189,13 @@ class TestRuby19Parser < RubyParserTestCase
     assert_parse rb, pt
   end
 
+  # def test_pipe_semicolon # HACK
+  #   rb = "a.b do | ; c | end"
+  #   pt = s(:iter, s(:call, s(:call, nil, :a), :b), 0)
+  # 
+  #   assert_parse rb, pt
+  # end
+
   # HACK: need to figure out the desired structure and get this working
   # def test_wtf
   #   # lambda -> f_larglist lambda_body

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: -2559988010
+  hash: 4039646929
   prerelease: 6
   segments: 
   - 3
   - 0
   - 0
   - a
-  - 5
-  version: 3.0.0.a5
+  - 6
+  version: 3.0.0.a6
 platform: ruby
 authors: 
 - Ryan Davis
@@ -38,7 +38,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-08-01 00:00:00 Z
+date: 2012-08-20 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor