ruby_parser 3.0.0.a8 → 3.0.0.a9

data/.autotest

@@ -13,6 +13,12 @@ Autotest.add_hook :initialize do |at|
   at.add_exception '.diff'
   at.add_exception 'rubycorpus'
 
+  dirs = Dir["corpus*"] + Dir["gauntlet*"]
+
+  dirs.each do |f|
+    at.add_exception f
+  end
+
   at.libs << ':../../minitest/dev/lib'
   at.testlib = "minitest/autorun"
 

data/History.txt

@@ -1,3 +1,43 @@
+=== 3.0.0.a9 / 2012-10-22
+
+Holy crap! I've hit a 99.92% success rate against 558k files! 492
+failed parses to go. NOW is the time to start testing ruby_parser
+against your code!
+
+* 16 minor enhancements:
+
+  * 1.9 parser: Finished magic encoding and utf-8 bom support.
+  * 1.9: Added leading dot support eg "a\n.b".
+  * 1.9: Added partial handling for *arg and opt=val args in block args.
+  * 1.9: OMFG Encoding is the worst thing ever. Overhauled encoding support once again. It is much better at doing last ditch guessing of the encoding of the source file. 
+  * 1.9: added String#grep for legacy support.
+  * Added Sexp#block_pass?
+  * Added ability to delete slow files when they timeout.
+  * Added block_dup_check and refactored grammar code to use it for block arg handling.
+  * Added in_lex_state?(*states) to clean up lexer code.
+  * Added tentative support for optional block args. THIS IS SUBJECT TO CHANGE!!
+  * Added toggleable debugging to StackState to make my life easier
+  * All untested uses have not moved but have been laced with poison. 
+  * Finally got good tests for most forms of double-block arg errors.
+  * Moved all _known_ uses of SyntaxError to RubyParser::SyntaxError
+  * f_block_optarg now always returns a block node.
+  * ruby_parse_extract_error uses Find.find instead of globbing so dot files aren't missed.
+
+* 12 bug fixes:
+
+  * 1.9: Completely ignore IndexError in unread_many because... you know... it sucks.
+  * 1.9: Fixed lex state after lexing ? in trinary.
+  * 1.9: Fixed lex state in some ternarys.
+  * 1.9: Fixed parsing of "1 ? b('') : 2\na d: 3"... ARE YOU NOT GLAD?!?!
+  * Fix Timeout differences between 1.8 and 1.9 :(
+  * Fixed emacs-style encodings to deal with no whitespace.
+  * Fixed error message for bad % codes. (whitequark)
+  * Fixed lexing of :a==>b vs :a===b vs :a==>b. P.S. Your space bar is broken. chump.
+  * Fixed lexing of rare numeric formats.
+  * Fixed magic comment encodings on DOS files ... :/
+  * Fixed ruby_parse_extract_error to exit non-zero on errors.
+  * Removed Symbol#is_argument and switch usage to is_arg?
+
 === 3.0.0.a8 / 2012-09-26
 
 * 1 bug fix:

data/README.txt

@@ -13,24 +13,19 @@ base types.
 
 As an example:
 
-  def conditional1(arg1)
-    if arg1 == 0 then
-      return 1
-    end
+  def conditional1 arg1
+    return 1 if arg1 == 0
     return 0
   end
 
 becomes:
 
-  s(:defn, :conditional1,
-   s(:args, :arg1),
-   s(:scope,
-    s(:block,
-     s(:if,
-      s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))),
+  s(:defn, :conditional1, s(:args, :arg1),
+    s(:if,
+      s(:call, s(:lvar, :arg1), :==, s(:lit, 0)),
       s(:return, s(:lit, 1)),
       nil),
-     s(:return, s(:lit, 0)))))
+    s(:return, s(:lit, 0)))
 
 == FEATURES/PROBLEMS:
 

data/Rakefile

@@ -171,6 +171,11 @@ task :debug => :isolate do
   end
 end
 
+task :debug_ruby do
+  file = ENV["F"] || ENV["FILE"]
+  sh "ruby19 -cwy #{file} 2>&1 | ./yuck.rb"
+end
+
 task :extract => :isolate do
   ENV["V"] ||= "19"
   Rake.application[:parser].invoke # this way we can have DEBUG set

data/bin/ruby_parse_extract_error

@@ -2,6 +2,8 @@
 
 $d ||= false
 $d ||= ENV["DELETE"]
+$t ||= false
+$t ||= ENV["DELETE_TIMEOUT"]
 $q ||= false
 $q ||= ENV["QUIET"]
 
@@ -13,6 +15,9 @@ ARGV.push "-" if ARGV.empty?
 class Racc::Parser
   def extract_defs
     ss = lexer.src
+
+    raise "can't access source. possible encoding issue" unless ss
+
     src = ss.string
     pre_error = src[0...ss.pos]
 
@@ -43,7 +48,15 @@ end
 
 def expand path
   if File.directory? path then
-    Dir.glob File.join path, "**/{*.rb,*.rake,Rakefile}"
+    require 'find'
+
+    files = []
+
+    Find.find(*Dir[path]) do |f|
+      files << f if File.file? f
+    end
+
+    files
   else
     Dir.glob path
   end
@@ -52,8 +65,6 @@ end
 def process_error parser
   defs = parser.extract_defs
 
-  orig_size = defs.size
-
   if parser.retest_for_errors defs then
     warn "Can't reproduce error with just methods, punting..."
     return
@@ -70,7 +81,7 @@ def process_error parser
     end
   end
 rescue RuntimeError, Racc::ParseError => e
-  warn "# error: #{e.message.strip}"
+  warn "# process error: #{e.message.strip}"
 end
 
 def process file
@@ -81,17 +92,21 @@ def process file
   parser.process(ruby, file)
   warn "good"
   File.unlink file if $d
+rescue Timeout::Error
+  $exit = 1
+  warn "TIMEOUT parsing #{file}. Skipping."
+  File.unlink file if $t
 rescue StandardError, SyntaxError, Racc::ParseError => e
+  $exit = 1
   warn ""
   warn "# error: #{e.message.strip}" unless $q
   warn ""
   return if $q
 
   process_error parser
-rescue Timeout::Error
-  warn "TIMEOUT parsing #{file}. Skipping."
 end
 
+$exit = 0
 $stdout.sync = true
 
 ARGV.each do |path|
@@ -100,3 +115,5 @@ ARGV.each do |path|
     process file
   end
 end
+
+exit $exit

data/lib/ruby18_parser.rb

@@ -3345,10 +3345,10 @@ end
 
 def _reduce_54(val, _values, result)
                       result = new_call nil, val[0].to_sym, val[1]
+
                       if val[2] then
-                        if result[0] == :block_pass then
-                          raise "both block arg and actual block given"
-                        end
+                        block_dup_check result, val[2]
+
                         result, operation = val[2], result
                         result.insert 1, operation
                       end
@@ -3364,6 +3364,14 @@ end
 
 def _reduce_56(val, _values, result)
                       result = new_call val[0], val[2].to_sym, val[3]
+                      raise "no2"
+
+                      if val[4] then
+                        block_dup_check result, val[4]
+
+                        val[2] << result
+                        result = val[2]
+                      end
                     
     result
 end
@@ -3376,10 +3384,11 @@ end
 
 def _reduce_58(val, _values, result)
                       result = new_call val[0], val[2].to_sym, val[3]
+                      raise "no3"
+
                       if val[4] then
-                        if result[0] == :block_pass then # REFACTOR
-                          raise "both block arg and actual block given"
-                        end
+                        block_dup_check result, val[4]
+
                         val[2] << result
                         result = val[2]
                       end
@@ -4497,6 +4506,8 @@ end
 
 def _reduce_293(val, _values, result)
                       call, iter = val[0], val[1]
+                      block_dup_check call, iter
+
                       iter.insert 1, call
                       result = iter
                     
@@ -4902,8 +4913,7 @@ def _reduce_361(val, _values, result)
 end
 
 def _reduce_362(val, _values, result)
-                      raise SyntaxError, "Both block arg and actual block given." if
-                        val[0] && val[0][0] == :blockpass
+                      block_dup_check val[0], val[1]
 
                       result = val[1]
                       result.insert 1, val[0]

data/lib/ruby18_parser.y

@@ -258,10 +258,10 @@ rule
                 | operation command_args cmd_brace_block
                     {
                       result = new_call nil, val[0].to_sym, val[1]
+
                       if val[2] then
-                        if result[0] == :block_pass then
-                          raise "both block arg and actual block given"
-                        end
+                        block_dup_check result, val[2]
+
                         result, operation = val[2], result
                         result.insert 1, operation
                       end
@@ -273,6 +273,14 @@ rule
                 | primary_value tDOT operation2 command_args cmd_brace_block
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
+                      raise "no2"
+
+                      if val[4] then
+                        block_dup_check result, val[4]
+
+                        val[2] << result
+                        result = val[2]
+                      end
                     }
                 | primary_value tCOLON2 operation2 command_args =tLOWEST
                     {
@@ -281,10 +289,11 @@ rule
                 | primary_value tCOLON2 operation2 command_args cmd_brace_block
                     {
                       result = new_call val[0], val[2].to_sym, val[3]
+                      raise "no3"
+
                       if val[4] then
-                        if result[0] == :block_pass then # REFACTOR
-                          raise "both block arg and actual block given"
-                        end
+                        block_dup_check result, val[4]
+
                         val[2] << result
                         result = val[2]
                       end
@@ -997,6 +1006,8 @@ rule
                 | method_call brace_block
                     {
                       call, iter = val[0], val[1]
+                      block_dup_check call, iter
+
                       iter.insert 1, call
                       result = iter
                     }
@@ -1287,8 +1298,7 @@ rule
 
       block_call: command do_block
                     {
-                      raise SyntaxError, "Both block arg and actual block given." if
-                        val[0] && val[0][0] == :blockpass
+                      block_dup_check val[0], val[1]
 
                       result = val[1]
                       result.insert 1, val[0]

data/lib/ruby19_parser.rb

@@ -2452,7 +2452,7 @@ racc_reduce_table = [
   4, 271, :_reduce_386,
   1, 272, :_reduce_none,
   2, 272, :_reduce_388,
-  1, 273, :_reduce_none,
+  1, 273, :_reduce_389,
   3, 273, :_reduce_390,
   1, 274, :_reduce_none,
   1, 274, :_reduce_none,
@@ -2593,7 +2593,7 @@ racc_reduce_table = [
   3, 266, :_reduce_527,
   3, 309, :_reduce_528,
   3, 310, :_reduce_529,
-  1, 267, :_reduce_none,
+  1, 267, :_reduce_530,
   3, 267, :_reduce_531,
   1, 307, :_reduce_532,
   3, 307, :_reduce_533,
@@ -3443,9 +3443,8 @@ end
 def _reduce_60(val, _values, result)
                       result = new_call nil, val[0].to_sym, val[1]
                       if val[2] then
-                        if result[0] == :block_pass then
-                          raise "both block arg and actual block given"
-                        end
+                        block_dup_check result, val[2]
+
                         result, operation = val[2], result
                         result.insert 1, operation
                       end
@@ -3460,7 +3459,13 @@ def _reduce_61(val, _values, result)
 end
 
 def _reduce_62(val, _values, result)
-                      result = new_call val[0], val[2].to_sym, val[3]
+                      recv, _, msg, args, block = val
+                      call = new_call recv, msg.to_sym, args
+
+                      block_dup_check call, block
+
+                      block.insert 1, call
+                      result = block
                     
     result
 end
@@ -3472,14 +3477,13 @@ def _reduce_63(val, _values, result)
 end
 
 def _reduce_64(val, _values, result)
-                      result = new_call val[0], val[2].to_sym, val[3]
-                      if val[4] then
-                        if result[0] == :block_pass then # REFACTOR
-                          raise "both block arg and actual block given"
-                        end
-                        val[2] << result
-                        result = val[2]
-                      end
+                      recv, _, msg, args, block = val
+                      call = new_call recv, msg.to_sym, args
+
+                      block_dup_check call, block
+
+                      block.insert 1, call
+                      result = block
                     
     result
 end
@@ -4356,7 +4360,7 @@ end
 # reduce 257 omitted
 
 def _reduce_258(val, _values, result)
-                      result = lexer.cmdarg.stack.dup
+                      result = lexer.cmdarg.stack.dup # TODO: smell?
                       lexer.cmdarg.push true
                     
     result
@@ -4546,7 +4550,7 @@ def _reduce_293(val, _values, result)
 end
 
 def _reduce_294(val, _values, result)
-                      raise "no3: #{val.inspect}"
+                      raise "no3\non#{val.inspect}"
                     
     result
 end
@@ -4565,7 +4569,8 @@ end
 
 def _reduce_297(val, _values, result)
                       call, iter = val[0], val[1]
-                      iter.insert 1, call
+                      block_dup_check call, iter
+                      iter.insert 1, call # FIX
                       result = iter
                     
     result
@@ -4887,49 +4892,49 @@ def _reduce_358(val, _values, result)
 end
 
 def _reduce_359(val, _values, result)
-                      raise "no9: #{val.inspect}"
+                      result = block_var val[0], val[3], nil
                     
     result
 end
 
 def _reduce_360(val, _values, result)
-                      raise "no10: #{val.inspect}"
+                      raise "no10\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_361(val, _values, result)
-                      raise "no11: #{val.inspect}"
+                      raise "no11\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_362(val, _values, result)
-                      raise "no12: #{val.inspect}"
+                      raise "no12\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_363(val, _values, result)
-                      raise "no13: #{val.inspect}"
+                      raise "no13\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_364(val, _values, result)
-                      raise "no14: #{val.inspect}"
+                      raise "no14\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_365(val, _values, result)
-                      raise "no15: #{val.inspect}"
+                      raise "no15\non: #{val.inspect}"
                     
     result
 end
 
 def _reduce_366(val, _values, result)
-                      raise "no16: #{val.inspect}"
+                      raise "no16\non: #{val.inspect}"
                     
     result
 end
@@ -4947,7 +4952,12 @@ def _reduce_368(val, _values, result)
 end
 
 def _reduce_369(val, _values, result)
-                      result = block_args19 val, "3"
+                      arg, _, opt, block = val
+
+                      result = arg
+                      result.concat opt[1..-1].map { |s| s[1] }
+                      result << "&#{block.last}".to_sym if block
+                      result << opt
                     
     result
 end
@@ -4995,7 +5005,12 @@ def _reduce_376(val, _values, result)
 end
 
 def _reduce_377(val, _values, result)
-                      result = block_args19 val, "11"
+                      opt, block = val
+
+                      result = s(:args)
+                      result.concat opt[1..-1].map { |s| s[1] }
+                      result << "&#{block.last}".to_sym if block
+                      result << opt
                     
     result
 end
@@ -5013,7 +5028,11 @@ def _reduce_379(val, _values, result)
 end
 
 def _reduce_380(val, _values, result)
-                      result = block_args19 val, "14"
+                      rest, _, args, block = val
+
+                      result = args
+                      result[1,0] = rest
+                      result << "&#{block.last}".to_sym if block
                     
     result
 end
@@ -5056,11 +5075,15 @@ def _reduce_388(val, _values, result)
     result
 end
 
-# reduce 389 omitted
+def _reduce_389(val, _values, result)
+                      result = [val[0]]
+                    
+    result
+end
 
 def _reduce_390(val, _values, result)
-                      result = val[0] << val[2]
-                      raise "no18: #{val.inspect}"
+                      result = val[0].concat val[2]
+                      raise "no18\non: #{val.inspect}"
                     
     result
 end
@@ -5135,8 +5158,12 @@ def _reduce_400(val, _values, result)
 end
 
 def _reduce_401(val, _values, result)
-                      raise SyntaxError, "Both block arg and actual block given." if
-                        val[0] && val[0][0] == :blockpass
+                      # TODO:
+                      # if (nd_type($1) == NODE_YIELD) {
+                      #     compile_error(PARSER_ARG "block given to yield");
+
+                      syntax_error "Both block arg and actual block given." if
+                        val[0].block_pass?
 
                       result = val[1]
                       result.insert 1, val[0]
@@ -5886,10 +5913,15 @@ def _reduce_529(val, _values, result)
     result
 end
 
-# reduce 530 omitted
+def _reduce_530(val, _values, result)
+                      result = s(:block, val[0])
+                    
+    result
+end
 
 def _reduce_531(val, _values, result)
-                      raise "no22: #{val.inspect}"
+                      result = val[0]
+                      result << val[2]
                     
     result
 end