ruby_parser 3.2.1 → 3.2.2

data/History.txt

@@ -1,3 +1,13 @@
+=== 3.2.2 / 2013-07-11
+
+* 5 bug fixes:
+
+  * 1.9/2.0: fixed assocs in return args. (presidentbeef)
+  * Fixed handling of parse error when class is nested in multiple defs. (whitequark)
+  * Fixed lexing of %w[] w/ funny whitespace separators. (whitequark)
+  * Fixed more call nodes that have trailing comma syntax. (presidentbeef)
+  * Fixed more call_args slippage.
+
 === 3.2.1 / 2013-07-03
 
 * 1 bug fix:

data/lib/ruby18_parser.rb

@@ -4677,18 +4677,23 @@ def _reduce_317(val, _values, result)
 end
 
 def _reduce_318(val, _values, result)
+                      result = lexer.lineno, self.in_def
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno, lexer.src.beginning_of_line?
                     
     result
 end
 
 def _reduce_319(val, _values, result)
+                      line, in_def = val[2]
+
                       result = new_defn val
+                      result[2].line line
+
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     
     result

data/lib/ruby18_parser.y

@@ -1124,16 +1124,21 @@ rule
                     }
                 | kDEF fname
                     {
+                      result = lexer.lineno, self.in_def
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno, lexer.src.beginning_of_line?
                     }
                     f_arglist bodystmt kEND
                     {
+                      line, in_def = val[2]
+
                       result = new_defn val
+                      result[2].line line
+
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF singleton dot_or_colon

data/lib/ruby19_parser.rb

@@ -4744,20 +4744,23 @@ def _reduce_321(val, _values, result)
 end
 
 def _reduce_322(val, _values, result)
+                      result = [lexer.lineno, self.in_def]
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno
                     
     result
 end
 
 def _reduce_323(val, _values, result)
+                      line, in_def = val[2]
+
                       result = new_defn val
-                      result[2].line val[2]
+                      result[2].line line
 
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     
     result

data/lib/ruby19_parser.y

@@ -1103,18 +1103,21 @@ rule
                     }
                 | kDEF fname
                     {
+                      result = [lexer.lineno, self.in_def]
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno
                     }
                     f_arglist bodystmt kEND
                     {
+                      line, in_def = val[2]
+
                       result = new_defn val
-                      result[2].line val[2]
+                      result[2].line line
 
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF singleton dot_or_colon

data/lib/ruby20_parser.rb

@@ -2225,7 +2225,7 @@ racc_reduce_table = [
   1, 146, :_reduce_none,
   3, 146, :_reduce_6,
   2, 146, :_reduce_none,
-  1, 149, :_reduce_none,
+  1, 149, :_reduce_8,
   0, 153, :_reduce_9,
   5, 149, :_reduce_10,
   4, 152, :_reduce_11,
@@ -3377,7 +3377,16 @@ end
 
 # reduce 7 omitted
 
-# reduce 8 omitted
+def _reduce_8(val, _values, result)
+                      result = val[0]
+
+                      # TODO: remove once I have more confidence this is fixed
+                      # result.each_of_type :call_args do |s|
+                      #   debug20 666, s, result
+                      # end
+                    
+    result
+end
 
 def _reduce_9(val, _values, result)
                       if (self.in_def || self.in_single > 0) then
@@ -4504,7 +4513,7 @@ end
 # reduce 243 omitted
 
 def _reduce_244(val, _values, result)
-                      result = val[0]
+                      result = args [val[0]]
                     
     result
 end
@@ -4517,7 +4526,6 @@ end
 
 def _reduce_246(val, _values, result)
                       result = args [array_to_hash(val[0])]
-                      result[0] = :array # TODO: switch to args?
                     
     result
 end
@@ -4750,6 +4758,7 @@ end
 
 def _reduce_291(val, _values, result)
                       result = val[1] || s(:array)
+                      result[0] = :array # aref_args is :args
                     
     result
 end
@@ -4916,7 +4925,6 @@ end
 def _reduce_318(val, _values, result)
                       self.comments.push self.lexer.comments
                       if (self.in_def || self.in_single > 0) then
-                        debug20 15
                         yyerror "class definition in method body"
                       end
                       self.env.extend
@@ -4986,20 +4994,23 @@ def _reduce_326(val, _values, result)
 end
 
 def _reduce_327(val, _values, result)
+                      result = [lexer.lineno, self.in_def]
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno
                     
     result
 end
 
 def _reduce_328(val, _values, result)
+                      line, in_def = val[2]
+
                       result = new_defn val
-                      result[2].line val[2]
+                      result[2].line line
 
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     
     result

data/lib/ruby20_parser.y

@@ -69,6 +69,14 @@ rule
                 | error top_stmt
 
         top_stmt: stmt
+                    {
+                      result = val[0]
+
+                      # TODO: remove once I have more confidence this is fixed
+                      # result.each_of_type :call_args do |s|
+                      #   debug20 666, s, result
+                      # end
+                    }
                 | klBEGIN
                     {
                       if (self.in_def || self.in_single > 0) then
@@ -574,7 +582,7 @@ rule
                 |   tMATCH   | tNMATCH | tGT      | tGEQ  | tLT    | tLEQ
                 |   tNEQ     | tLSHFT  | tRSHFT   | tPLUS | tMINUS | tSTAR2
                 |   tSTAR    | tDIVIDE | tPERCENT | tPOW  | tDSTAR | tBANG   | tTILDE
-                |   tUPLUS   | tUMINUS | tUBANG   | tAREF | tASET  | tBACK_REF2 
+                |   tUPLUS   | tUMINUS | tUBANG   | tAREF | tASET  | tBACK_REF2
 
         reswords: k__LINE__ | k__FILE__ | k__ENCODING__ | klBEGIN | klEND
                 | kALIAS    | kAND      | kBEGIN        | kBREAK  | kCASE
@@ -682,7 +690,7 @@ rule
                     {
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
                       debug20 12, val, result
-                    } 
+                    }
                | tUPLUS arg
                     {
                       result = new_call val[1], :"+@"
@@ -790,7 +798,7 @@ rule
        aref_args: none
                 | args trailer
                     {
-                      result = val[0]
+                      result = args [val[0]]
                     }
                 | args tCOMMA assocs trailer
                     {
@@ -799,7 +807,6 @@ rule
                 | assocs trailer
                     {
                       result = args [array_to_hash(val[0])]
-                      result[0] = :array # TODO: switch to args?
                     }
 
       paren_args: tLPAREN2 opt_call_args rparen
@@ -964,6 +971,7 @@ rule
                 | tLBRACK aref_args tRBRACK
                     {
                       result = val[1] || s(:array)
+                      result[0] = :array # aref_args is :args
                     }
                 | tLBRACE assoc_list tRCURLY
                     {
@@ -1077,7 +1085,6 @@ rule
                     {
                       self.comments.push self.lexer.comments
                       if (self.in_def || self.in_single > 0) then
-                        debug20 15
                         yyerror "class definition in method body"
                       end
                       self.env.extend
@@ -1129,18 +1136,21 @@ rule
                     }
                 | kDEF fname
                     {
+                      result = [lexer.lineno, self.in_def]
+
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = lexer.lineno
                     }
                     f_arglist bodystmt kEND
                     {
+                      line, in_def = val[2]
+
                       result = new_defn val
-                      result[2].line val[2]
+                      result[2].line line
 
                       self.env.unextend
-                      self.in_def = false
+                      self.in_def = in_def
                       self.lexer.comments # we don't care about comments in the body
                     }
                 | kDEF singleton dot_or_colon

data/lib/ruby_lexer.rb

@@ -616,7 +616,11 @@ class RubyLexer
         t = Regexp.escape term
         x = Regexp.escape(paren) if paren && paren != "\000"
         re = if qwords then
-               /[^#{t}#{x}\#\0\\\n\ ]+|./ # |. to pick up whatever
+               if RUBY19 then
+                 /[^#{t}#{x}\#\0\\\s]+|./ # |. to pick up whatever
+               else
+                 /[^#{t}#{x}\#\0\\\s\v]+|./ # argh. 1.8's \s doesn't pick up \v
+               end
              else
                /[^#{t}#{x}\#\0\\]+|./
              end

data/lib/ruby_parser_extras.rb

@@ -111,7 +111,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = "3.2.1" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.2.2" unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -596,7 +596,7 @@ module RubyParserStuff
     # TODO: need a test with f(&b) { } to produce warning
 
     args ||= s(:arglist)
-    args[0] = :arglist if [:array, :call_args].include? args.first
+    args[0] = :arglist if [:args, :array, :call_args].include? args.first
     args = s(:arglist, args) unless args.first == :arglist
 
     # HACK quick hack to make this work quickly... easy to clean up above
@@ -655,7 +655,7 @@ module RubyParserStuff
   end
 
   def new_defn val
-    (_, line), name, args, body = val[0], val[1], val[3], val[4]
+    (_, line), name, _, args, body, * = val
     body ||= s(:nil)
 
     result = s(:defn, name.to_sym, args)
@@ -916,8 +916,7 @@ module RubyParserStuff
 
     args ||= s(:arglist)
 
-    # TODO: I can prolly clean this up
-    args[0] = :arglist       if args.first == :array
+    args[0] = :arglist if [:call_args, :array].include?(args[0])
     args = s(:arglist, args) unless args.first == :arglist
 
     return s(:yield, *args[1..-1])
@@ -1075,7 +1074,9 @@ module RubyParserStuff
       raise SyntaxError, "block argument should not be given" if
         node[0] == :block_pass
 
+      node[0] = :array if node[0] == :call_args
       node = node.last if node[0] == :array && node.size == 2
+
       # HACK matz wraps ONE of the FOUR splats in a newline to
       # distinguish. I use paren for now. ugh
       node = s(:svalue, node) if node[0] == :splat and not node.paren

data/test/test_ruby_lexer.rb

@@ -2231,14 +2231,6 @@ class TestRubyLexer < Minitest::Test
                    :tSTRING_END,     nil)
   end
 
-  def test_yylex_string_pct_w_tab
-    util_lex_token("%w[abc\tdef]",
-                   :tQWORDS_BEG,      "%w[",
-                   :tSTRING_CONTENT, "abc\tdef",
-                   :tSPACE,              nil,
-                   :tSTRING_END,     nil)
-  end
-
   def test_yylex_string_single
     util_lex_token("'string'",
                    :tSTRING, "string")
@@ -2375,6 +2367,24 @@ class TestRubyLexer < Minitest::Test
                    :tFLOAT, 0.0)
   end
 
+  def test_pct_w_backslashes
+    ["\t", "\n", "\r", "\v", "\f"].each do |char|
+      next if !RubyLexer::RUBY19 and char == "\v"
+
+      assert_lex("%w[foo#{char}bar]",
+                 s(:array, s(:str, "foo"), s(:str, "bar")),
+
+                 :tQWORDS_BEG,     "%w[", :expr_beg, 0, 0,
+                 :tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
+                 :tSPACE,          nil,   :expr_beg, 0, 0,
+                 :tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
+                 :tSPACE,          nil,   :expr_beg, 0, 0,
+                 :tSTRING_END,     nil,   :expr_end, 0, 0)
+    end
+
+    # flunk "Not yet"
+  end
+
   ############################################################
 
   def util_bad_token s, *args

data/test/test_ruby_parser.rb

@@ -1236,6 +1236,15 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_aref_args_lit_assocs
+    skip if ruby18
+
+    rb = "[1, 2 => 3]"
+    pt = s(:array, s(:lit, 1), s(:hash, s(:lit, 2), s(:lit, 3)))
+
+    assert_parse rb, pt
+  end
+
   def test_BEGIN
     rb = "BEGIN { 42 }"
     pt = s(:iter, s(:preexe), s(:args), s(:lit, 42))
@@ -1429,6 +1438,13 @@ module TestRubyParserShared
 
     assert_parse rb, pt
   end
+
+  def test_fubar_nesting
+    err = "class definition in method body"
+
+    assert_syntax_error "def a; class B; end; end", err
+    assert_syntax_error "def a; def b; end; class B; end; end", err
+  end
 end
 
 module TestRubyParserShared1920
@@ -1667,6 +1683,13 @@ module TestRubyParserShared1920
     assert_parse rb, pt
   end
 
+  def test_method_call_trailing_comma
+    rb = "a.f(1,)"
+    pt = s(:call, s(:call, nil, :a), :f, s(:lit, 1))
+
+    assert_parse rb, pt
+  end
+
   def test_call_assoc_trailing_comma
     rb = "f(1=>2,)"
     pt = s(:call, nil, :f, s(:hash, s(:lit, 1), s(:lit, 2)))
@@ -1674,6 +1697,13 @@ module TestRubyParserShared1920
     assert_parse rb, pt
   end
 
+  def test_method_call_assoc_trailing_comma
+    rb = "a.f(1=>2,)"
+    pt = s(:call, s(:call, nil, :a), :f, s(:hash, s(:lit, 1), s(:lit, 2)))
+
+    assert_parse rb, pt
+  end
+
   def test_call_args_assoc_trailing_comma
     rb = "f(1, 2=>3,)"
     pt = s(:call, nil, :f, s(:lit, 1), s(:hash, s(:lit, 2), s(:lit, 3)))
@@ -1695,6 +1725,80 @@ module TestRubyParserShared1920
     assert_parse rb, pt
   end
 
+  def test_return_call_assocs
+    rb = "return y(z:1)"
+    pt = s(:return, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "return y z:1"
+    pt = s(:return, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "return y(z=>1)"
+    pt = s(:return, s(:call, nil, :y, s(:hash, s(:call, nil, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "return y :z=>1"
+    pt = s(:return, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "return 1, :z => 1"
+    pt = s(:return,
+           s(:array,
+             s(:lit, 1),
+             s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "return 1, :z => 1, :w => 2"
+    pt = s(:return,
+           s(:array,
+             s(:lit, 1),
+             s(:hash, s(:lit, :z), s(:lit, 1), s(:lit, :w), s(:lit, 2))))
+
+    assert_parse rb, pt
+  end
+
+  def test_yield_call_assocs
+    rb = "yield y(z:1)"
+    pt = s(:yield, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "yield y z:1"
+    pt = s(:yield, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "yield y(z=>1)"
+    pt = s(:yield, s(:call, nil, :y, s(:hash, s(:call, nil, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "yield y :z=>1"
+    pt = s(:yield, s(:call, nil, :y, s(:hash, s(:lit, :z), s(:lit, 1))))
+
+    assert_parse rb, pt
+
+    rb = "yield 1, :z => 1"
+    pt = s(:yield,
+           s(:lit, 1),
+           s(:hash, s(:lit, :z), s(:lit, 1)))
+
+    assert_parse rb, pt
+
+    rb = "yield 1, :z => 1, :w => 2"
+    pt = s(:yield,
+           s(:lit, 1),
+           s(:hash, s(:lit, :z), s(:lit, 1), s(:lit, :w), s(:lit, 2)))
+
+    assert_parse rb, pt
+  end
+
   def test_call_assoc_new
     rb = "f(a:3)"
     pt = s(:call, nil, :f, s(:hash, s(:lit, :a), s(:lit, 3)))
@@ -1821,6 +1925,13 @@ module TestRubyParserShared1920
 
     assert_parse rb, pt
   end
+
+  def test_call_array_lit_inline_hash
+    rb = "a([:b, :c => 1])"
+    pt = s(:call, nil, :a, s(:array, s(:lit, :b), s(:hash, s(:lit, :c), s(:lit, 1))))
+
+    assert_parse rb, pt
+  end
 end
 
 class TestRubyParser < Minitest::Test
@@ -2244,6 +2355,13 @@ class TestRuby19Parser < RubyParserTestCase
     assert_parse rb, pt
   end
 
+  def test_parse_opt_call_args_lit_comma
+    rb = "1[2,]"
+    pt = s(:call, s(:lit, 1), :[], s(:lit, 2))
+
+    assert_parse rb, pt
+  end
+
   def test_bug_hash_args
     rb = "foo(:bar, baz: nil)"
     pt = s(:call, nil, :foo,

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 3
   - 2
-  - 1
-  version: 3.2.1
+  - 2
+  version: 3.2.2
 platform: ruby
 authors: 
 - Ryan Davis
@@ -36,7 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2013-07-03 00:00:00 Z
+date: 2013-07-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor