ruby_parser 3.1.3 → 3.2.0

data/.autotest

@@ -11,7 +11,12 @@ Autotest.add_hook :initialize do |at|
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
   at.add_exception '.diff'
+  at.add_exception '.output'
   at.add_exception 'rubycorpus'
+  at.add_exception "lib/ruby18_parser.rb"
+  at.add_exception "lib/ruby19_parser.rb"
+  at.add_exception "lib/ruby20_parser.rb"
+  at.add_exception "lib/gauntlet_rubyparser.rb"
 
   dirs = Dir["corpus*"] + Dir["gauntlet*"]
 
@@ -23,6 +28,8 @@ Autotest.add_hook :initialize do |at|
   at.testlib = "minitest/autorun"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
+    re = %r%^test/test_#{File.basename(f, '.y').gsub(/(\d+)/, '')}.rb$%
+    at.files_matching re
     at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
   end
 
@@ -34,7 +41,7 @@ Autotest.add_hook :initialize do |at|
     at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
   end
 
-  %w(TestRuby18Parser TestRuby19Parser TestParseTree).each do |klass| # HACK
+  %w(TestRuby18Parser TestRuby19Parser TestRuby20Parser TestParseTree).each do |klass| # HACK
     at.extra_class_map[klass] = "test/test_ruby_parser.rb"
   end
 end
@@ -45,6 +52,12 @@ end
 
 class Autotest
   def ruby
-    File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
+    File.expand_path "~/.multiruby/install/1.9.3-p194/bin/ruby"
   end
-end if ENV['ONENINE']
+end if ENV['R19']
+
+class Autotest
+  def ruby
+    File.expand_path "~/.multiruby/install/2.0.0-p195/bin/ruby"
+  end
+end if ENV['R20']

data/History.txt

@@ -1,3 +1,69 @@
+=== 3.2.0 / 2013-07-02
+
+* 1 major enhancement:
+
+  * Added (rough draft) 2.0 support. Still missing some small / rare things.
+
+* 12 minor enhancements:
+
+  * Added %i(symbol-names...) support. (%I too)
+  * Added 140 more tests, jumping test count from 1376 to 2143. Yay for test reuse!
+  * Added RubyLexer#brace_nest.
+  * Added compare20 rake task to diff the grammar architecture against MRI.
+  * Added lpar_beg and paren_nest to lexer to track state of parens in stabbies
+  * Added shadow nodes for scoped block args.
+  * Compound RubyParser now defaults to 2.0.
+  * Fixed rake to < 10, because 10's file dependency handling is so very broken.
+  * Made it possible to specify version in bin/ruby_parse_extract_error w/ -v 18|19|20
+  * Refactored to RubyParserStuff::ENCODING_ORDER to allow custom tweaking of encoding guessing. (samlown)
+  * Switched `rake debug` to default to 2.0.
+  * Translated some fixes across 1.8 and 1.9 from 2.0.
+
+* 42 bug fixes:
+
+  * 2.0: Fixed a number of block args scenarios w/ kwargs
+  * 2.0: Fixed args_tail mismatching against lexer.
+  * 2.0: Fixed assocs to return a hash node.
+  * 2.0: Fixed f_block_kw production.
+  * 2.0: Fixed f_block_kwarg production.
+  * 2.0: Fixed handling of stabby proc args in parens.
+  * 2.0: Fixed lexing of kwsplat nodes.
+  * 2.0: Implemented kwsplat nodes.
+  * Added tUBANG to lexer.
+  * Apparently ruby doesn't warn for escaped octal that goes above 7. wtf.
+  * Cleaned up a LOT of arg handling (block and defn, not calls) by using #args.
+  * ESC_RE is set to unicode. This seems problematic. *shrug*
+  * Either found a bug in MRI and/or fixed paren_nest count for []=? methods.
+  * Extended IDENT_CHAR_RE on 1.9+ to top out at \u{10ffff}... because we NEED a million unicode chars.
+  * Fixed % strings with interpolation.
+  * Fixed BEGIN {} to return a sexp.
+  * Fixed a[] += b. (stormbrew)
+  * Fixed arg_blk_pass to allow for sub-args nodes.
+  * Fixed assignable to allow for sexps to be passed to it.
+  * Fixed assoc args in 1.9.
+  * Fixed block_command and block_call (eg a.b c d) to #to_sym their args properly.
+  * Fixed bug in compound RubyParser so it rescues RubyParser::SyntaxError.
+  * Fixed env registration of cdecls.
+  * Fixed lex value for { when expr_endfn.
+  * Fixed lex_state for close paren/brace/bracket.
+  * Fixed lex_state transition for 1.9 if we lexed a defn name. Only 1.8 is odd.
+  * Fixed lexer problem with state mgmt of identifiers that also have registered var name.
+  * Fixed lexing of "1 *\n" to have the correct lex_state.
+  * Fixed lexing of heredocs vs chevron for some lex_states.
+  * Fixed op_asgn nodes to #to_sym their args properly.
+  * Fixed optional value block args.
+  * Fixed parsing of __ENCODING__ on ruby 1.8 (vcall).
+  * Fixed some oddity where 1.9 lexing was blowing up on "0o". Seems invalid now.
+  * Fixed strings with escaped octals > 128. Also... wtf.
+  * Fixed support for empty symbol (wtf?).
+  * Lexer is now declared UTF-8 internally. Hopefully this will fix the encoding mess.
+  * Made UTF_8 the default guess on encodings when it isn't explicit.
+  * Parsing of __ENCODING__ on ruby 1.9+ (in ruby 1.9+) is now colon2 sexp. (whitequark)
+  * Renamed RubyLexer#nest to string_nest
+  * RubyLexer#unescape ignores bad octal/hex and returns unicode strings.
+  * Switched a number of lexical constructs to use IDENT_CHAR_RE instead of \w. I wish there were something cleaner for regexps + unicode.
+  * Switched ruby_parse_extract_error to use binread.
+
 === 3.1.3 / 2013-04-09
 
 * 2 bug fixes:

data/Manifest.txt

@@ -10,6 +10,8 @@ lib/ruby18_parser.rb
 lib/ruby18_parser.y
 lib/ruby19_parser.rb
 lib/ruby19_parser.y
+lib/ruby20_parser.rb
+lib/ruby20_parser.y
 lib/ruby_lexer.rb
 lib/ruby_parser.rb
 lib/ruby_parser_extras.rb

data/Rakefile

@@ -8,6 +8,7 @@ Hoe.plugin :racc
 Hoe.plugin :isolate
 
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
+Hoe.add_include_dirs "../../minitest/dev/lib"
 
 Hoe.spec 'ruby_parser' do
   developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
@@ -15,10 +16,12 @@ Hoe.spec 'ruby_parser' do
   self.rubyforge_name = 'parsetree'
 
   dependency 'sexp_processor', '~> 4.1'
+  dependency 'rake', '< 10', :developer
 
   if plugin? :perforce then
     self.perforce_ignore << "lib/ruby18_parser.rb"
     self.perforce_ignore << "lib/ruby19_parser.rb"
+    self.perforce_ignore << "lib/ruby20_parser.rb"
   end
 
   self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
@@ -26,6 +29,7 @@ end
 
 file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
 file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
+file "lib/ruby20_parser.rb" => "lib/ruby20_parser.y"
 
 task :clean do
   rm_rf(Dir["**/*~"] +
@@ -50,16 +54,6 @@ task :compare do
   system 'find -d unit -type d -empty -exec rmdir {} \;'
 end
 
-desc "Compares PT to RP and stops on first failure"
-task :find_bug do
-  files = Dir["unit/**/*.rb"]
-  puts "Parsing #{files.size} files"
-  files.each do |file|
-    puts file
-    sh "./cmp.rb -q #{file}"
-  end
-end
-
 task :sort do
   sh 'grepsort "^ +def" lib/ruby_lexer.rb'
   sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
@@ -121,6 +115,7 @@ task :isolate => :phony
 
 file "lib/ruby18_parser.rb" => :isolate
 file "lib/ruby19_parser.rb" => :isolate
+file "lib/ruby20_parser.rb" => :isolate
 
 task :compare18 do
   sh "./yack.rb lib/ruby18_parser.output > racc18.txt"
@@ -138,18 +133,29 @@ task :compare19 do
   sh "diff -du racc19.txt yacc19.txt | wc -l"
 end
 
+task :compare20 do
+  sh "./yack.rb lib/ruby20_parser.output > racc20.txt"
+  sh "./yack.rb parse20.output > yacc20.txt"
+  sh "diff -du racc20.txt yacc20.txt || true"
+  puts
+  sh "diff -du racc20.txt yacc20.txt | wc -l"
+end
+
 task :debug => :isolate do
-  ENV["V"] ||= "19"
+  ENV["V"] ||= "20"
   Rake.application[:parser].invoke # this way we can have DEBUG set
 
   $: << "lib"
   require 'ruby_parser'
   require 'pp'
 
-  parser = if ENV["V"] == "18" then
+  parser = case ENV["V"]
+           when "18" then
              Ruby18Parser.new
-           else
+           when "19" then
              Ruby19Parser.new
+           else
+             Ruby20Parser.new
            end
 
   time = (ENV["RP_TIMEOUT"] || 10).to_i
@@ -174,9 +180,13 @@ task :debug => :isolate do
   end
 end
 
+def ruby20
+  "/Users/ryan/.multiruby/install/2.0.0-p195/bin/ruby"
+end
+
 task :debug_ruby do
   file = ENV["F"] || ENV["FILE"]
-  sh "ruby19 -cwy #{file} 2>&1 | ./yuck.rb"
+  sh "#{ruby20} -cwy #{file} 2>&1 | ./yuck.rb"
 end
 
 task :extract => :isolate do
@@ -189,7 +199,7 @@ task :extract => :isolate do
 end
 
 task :bugs do
-  sh "for f in bug*.rb ; do rake19 debug F=$f && rm $f ; done"
+  sh "for f in bug*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
 end
 
 # vim: syntax=Ruby

data/bin/ruby_parse_extract_error

@@ -8,11 +8,31 @@ $m ||= false
 $m ||= ENV["MOVE_TIMEOUT"]
 $q ||= false
 $q ||= ENV["QUIET"]
+$v ||= ENV["V"] || "20"
 
 require 'rubygems'
 require 'ruby_parser'
 require 'fileutils'
 
+$parser_class = case $v
+                when "18" then
+                  Ruby18Parser
+                when "19" then
+                  Ruby19Parser
+                when "20" then
+                  Ruby20Parser
+                else
+                  abort "Unknown version #{$v.inspect}. Needs to be 18, 19, or 20"
+                end
+
+class IO
+  RUBY19 = "<3".respond_to? :encoding
+
+  class << self
+    alias :binread :read unless RUBY19
+  end
+end
+
 ARGV.push "-" if ARGV.empty?
 
 class Racc::Parser
@@ -88,11 +108,11 @@ rescue RuntimeError, Racc::ParseError => e
 end
 
 def process file
-  ruby = file == "-" ? $stdin.read : File.read(file)
+  ruby = file == "-" ? $stdin.binread : File.binread(file)
   time = (ENV["RP_TIMEOUT"] || 10).to_i
 
   $stderr.print "# Validating #{file}: "
-  parser = Ruby19Parser.new
+  parser = $parser_class.new
   parser.process(ruby, file, time)
   warn "good"
   File.unlink file if $d

data/lib/ruby18_parser.rb

@@ -3163,8 +3163,7 @@ def _reduce_20(val, _values, result)
 end
 
 def _reduce_21(val, _values, result)
-                      result = new_iter s(:preexe), nil, val[3] # TODO: add test?
-                      result = nil # TODO: since it isn't supposed to go in the AST
+                      result = new_iter s(:preexe), nil, val[3]
                     
     result
 end
@@ -3203,13 +3202,13 @@ def _reduce_26(val, _values, result)
 end
 
 def _reduce_27(val, _values, result)
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     
     result
 end
 
 def _reduce_28(val, _values, result)
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     
     result
 end
@@ -3822,7 +3821,7 @@ end
 
 def _reduce_177(val, _values, result)
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
-                      val[2][0] = :arglist
+                      val[2][0] = :arglist if val[2]
                     
     result
 end
@@ -3840,7 +3839,7 @@ def _reduce_179(val, _values, result)
 end
 
 def _reduce_180(val, _values, result)
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     
     result
 end
@@ -5249,6 +5248,7 @@ end
 
 def _reduce_419(val, _values, result)
                       result = lexer.lex_strterm
+
                       lexer.lex_strterm = nil
                       lexer.lex_state = :expr_beg
                     
@@ -5263,32 +5263,44 @@ def _reduce_420(val, _values, result)
 end
 
 def _reduce_421(val, _values, result)
-                      result = lexer.lex_strterm
+                      result = [lexer.lex_strterm, lexer.brace_nest, lexer.string_nest]
+
                       lexer.lex_strterm = nil
-                      lexer.lex_state = :expr_beg
+                      lexer.brace_nest  = 0
+                      lexer.string_nest = 0
+
                       lexer.cond.push false
                       lexer.cmdarg.push false
+
+                      lexer.lex_state   = :expr_beg
                     
     result
 end
 
 def _reduce_422(val, _values, result)
-                      lexer.lex_strterm = val[1]
+                      _, memo, stmt, _ = val
+
+                      lex_strterm, brace_nest, string_nest = memo
+
+                      lexer.lex_strterm = lex_strterm
+                      lexer.brace_nest  = brace_nest
+                      lexer.string_nest = string_nest
+
                       lexer.cond.lexpop
                       lexer.cmdarg.lexpop
 
-                      case val[2]
+                      case stmt
                       when Sexp then
-                        case val[2][0]
+                        case stmt[0]
                         when :str, :dstr, :evstr then
-                          result = val[2]
+                          result = stmt
                         else
-                          result = s(:evstr, val[2])
+                          result = s(:evstr, stmt)
                         end
                       when nil then
                         result = s(:evstr)
                       else
-                        raise "unknown rescue body: #{val[2].inspect}"
+                        raise "unknown string body: #{stmt.inspect}"
                       end
                     
     result
@@ -5343,7 +5355,7 @@ def _reduce_433(val, _values, result)
                       when :dstr then
                         result[0] = :dsym
                       when :str then
-                        result = s(:lit, result.last.intern)
+                        result = s(:lit, result.last.to_sym)
                       else
                         result = s(:dsym, "", result)
                       end

data/lib/ruby18_parser.y

@@ -128,8 +128,7 @@ rule
                     }
                     tLCURLY compstmt tRCURLY
                     {
-                      result = new_iter s(:preexe), nil, val[3] # TODO: add test?
-                      result = nil # TODO: since it isn't supposed to go in the AST
+                      result = new_iter s(:preexe), nil, val[3]
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
@@ -156,11 +155,11 @@ rule
                     }
                 | primary_value tDOT tIDENTIFIER tOP_ASGN command_call
                     {
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     }
                 | primary_value tDOT tCONSTANT tOP_ASGN command_call
                     {
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     }
                 | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_call
                     {
@@ -522,7 +521,7 @@ rule
                 | primary_value tLBRACK2 aref_args tRBRACK tOP_ASGN arg
                     {
                       result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
-                      val[2][0] = :arglist
+                      val[2][0] = :arglist if val[2]
                     }
                 | primary_value tDOT tIDENTIFIER tOP_ASGN arg
                     {
@@ -534,7 +533,7 @@ rule
                     }
                 | primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg
                     {
-                      result = s(:op_asgn, val[0], val[4], val[2], val[3])
+                      result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
                     {
@@ -1545,6 +1544,7 @@ xstring_contents: none
                 | tSTRING_DVAR
                     {
                       result = lexer.lex_strterm
+
                       lexer.lex_strterm = nil
                       lexer.lex_state = :expr_beg
                     }
@@ -1555,30 +1555,42 @@ xstring_contents: none
                     }
                 | tSTRING_DBEG
                     {
-                      result = lexer.lex_strterm
+                      result = [lexer.lex_strterm, lexer.brace_nest, lexer.string_nest]
+
                       lexer.lex_strterm = nil
-                      lexer.lex_state = :expr_beg
+                      lexer.brace_nest  = 0
+                      lexer.string_nest = 0
+
                       lexer.cond.push false
                       lexer.cmdarg.push false
+
+                      lexer.lex_state   = :expr_beg
                     }
                     compstmt tRCURLY
                     {
-                      lexer.lex_strterm = val[1]
+                      _, memo, stmt, _ = val
+
+                      lex_strterm, brace_nest, string_nest = memo
+
+                      lexer.lex_strterm = lex_strterm
+                      lexer.brace_nest  = brace_nest
+                      lexer.string_nest = string_nest
+
                       lexer.cond.lexpop
                       lexer.cmdarg.lexpop
 
-                      case val[2]
+                      case stmt
                       when Sexp then
-                        case val[2][0]
+                        case stmt[0]
                         when :str, :dstr, :evstr then
-                          result = val[2]
+                          result = stmt
                         else
-                          result = s(:evstr, val[2])
+                          result = s(:evstr, stmt)
                         end
                       when nil then
                         result = s(:evstr)
                       else
-                        raise "unknown rescue body: #{val[2].inspect}"
+                        raise "unknown string body: #{stmt.inspect}"
                       end
                     }
 
@@ -1587,7 +1599,6 @@ xstring_contents: none
                 | tCVAR { result = s(:cvar, val[0].to_sym) }
                 | backref
 
-
           symbol: tSYMBEG sym
                     {
                       lexer.lex_state = :expr_end
@@ -1612,7 +1623,7 @@ xstring_contents: none
                       when :dstr then
                         result[0] = :dsym
                       when :str then
-                        result = s(:lit, result.last.intern)
+                        result = s(:lit, result.last.to_sym)
                       else
                         result = s(:dsym, "", result)
                       end