ruby_parser 2.0.5 → 3.0.0.a1

data/test/test_ruby_lexer.rb

@@ -1,15 +1,21 @@
 #!/usr/local/bin/ruby
 
 require 'rubygems'
+gem "minitest"
+
 require 'minitest/autorun'
 require 'ruby_lexer'
-require 'ruby_parser'
+require 'ruby18_parser'
 
 class TestRubyLexer < MiniTest::Unit::TestCase
   alias :deny :refute
 
   def setup
-    p = RubyParser.new
+    setup_lexer Ruby18Parser
+  end
+
+  def setup_lexer parser_class
+    p = parser_class.new
     @lex = p.lexer
     @lex.src = "blah blah"
     @lex.lex_state = :expr_beg
@@ -125,6 +131,38 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "=>", :tASSOC, "=>"
   end
 
+  def test_yylex_label__18
+    util_lex_token("{a:",
+                   :tLBRACE,     "{",
+                   :tIDENTIFIER, "a",
+                   :tSYMBEG,     ":")
+  end
+
+  def test_yylex_label_in_params__18
+    util_lex_token("foo(a:",
+                   :tIDENTIFIER, "foo",
+                   :tLPAREN2,    "(",
+                   :tIDENTIFIER, "a",
+                   :tSYMBEG,     ":")
+  end
+
+  def test_yylex_label__19
+    setup_lexer Ruby19Parser
+
+    util_lex_token("{a:",
+                   :tLBRACE, "{",
+                   :tLABEL,  "a")
+  end
+
+  def test_yylex_label_in_params__19
+    setup_lexer Ruby19Parser
+
+    util_lex_token("foo(a:",
+                   :tIDENTIFIER, "foo",
+                   :tLPAREN2,    "(",
+                   :tLABEL,      "a")
+  end
+
   def test_yylex_back_ref
     util_lex_token("[$&, $`, $', $+]",
                    :tLBRACK,   "[",
@@ -255,6 +293,11 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     assert_equal "=begin blah\nblah\n=end\n", @lex.comments
   end
 
+  def test_yylex_comment_end_space_and_text
+    util_lex_token("=begin blah\nblah\n=end blab\n")
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+  end
+
   def test_yylex_comment_eos
     util_lex_token("# comment")
   end
@@ -605,7 +648,6 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_CONTENT, "blah\nblah\n",
-                   :tSTRING_CONTENT, "",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
   end
@@ -623,7 +665,6 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_CONTENT, "blah\nblah\n",
-                   :tSTRING_CONTENT, "",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
   end
@@ -800,14 +841,30 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_bad_token "0d42__24"
   end
 
-  def test_yylex_integer_eh_a
-    util_lex_token '?a', :tINTEGER, 97
+  def test_yylex_question_eh_a__18
+    @lex = RubyLexer.new 18
+
+    util_lex_token "?a", :tINTEGER, 97
   end
 
-  def test_yylex_integer_eh_escape_M_escape_C
+  def test_yylex_question_eh_a__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token '?a', :tSTRING, "a"
+  end
+
+  def test_yylex_question_eh_escape_M_escape_C__18
+    @lex = RubyLexer.new 18
+
     util_lex_token '?\M-\C-a', :tINTEGER, 129
   end
 
+  def test_yylex_question_eh_escape_M_escape_C__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
+  end
+
   def test_yylex_integer_hex
     util_lex_token "0x2a", :tINTEGER, 42
   end
@@ -1023,7 +1080,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   def test_yylex_open_square_bracket_meth
     util_lex_token("m[3]",
                    :tIDENTIFIER, "m",
-                   "[", "[",
+                   :tLBRACK2, "[",
                    :tINTEGER, 3,
                    :tRBRACK, "]")
   end
@@ -1084,10 +1141,18 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tINTEGER, 42)
   end
 
-  def test_yylex_question
+  def test_yylex_question__18
+    @lex = RubyLexer.new 18
+
     util_lex_token "?*", :tINTEGER, 42
   end
 
+  def test_yylex_question__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token "?*", :tSTRING, "*"
+  end
+
   def test_yylex_question_bad_eos
     util_bad_token "?"
   end
@@ -1101,7 +1166,9 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "?\f", :tEH, "?"
   end
 
-  def test_yylex_question_ws_backslashed
+  def test_yylex_question_ws_backslashed__18
+    @lex = RubyLexer.new 18
+
     @lex.lex_state = :expr_beg
     util_lex_token "?\\ ", :tINTEGER, 32
     @lex.lex_state = :expr_beg
@@ -1116,6 +1183,23 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "?\\f", :tINTEGER, 12
   end
 
+  def test_yylex_question_ws_backslashed__19
+    @lex = RubyLexer.new 19
+
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\ ", :tSTRING, " "
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\n", :tSTRING, "\n"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\t", :tSTRING, "\t"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\v", :tSTRING, "\v"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\r", :tSTRING, "\r"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\f", :tSTRING, "\f"
+  end
+
   def test_yylex_rbracket
     util_lex_token "]", :tRBRACK, "]"
   end
@@ -1609,7 +1693,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w
     util_bad_token("%w[s1 s2 ",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s2",
@@ -1618,7 +1702,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_bs_nl
     util_lex_token("%w[s1 \\\ns2]",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "\ns2",
@@ -1628,7 +1712,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_bs_sp
     util_lex_token("%w[s\\ 1 s\\ 2]",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s 1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s 2",
@@ -1638,7 +1722,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_tab
     util_lex_token("%w[abc\tdef]",
-                   :tAWORDS_BEG,      "%w[",
+                   :tQWORDS_BEG,      "%w[",
                    :tSTRING_CONTENT, "abc\tdef",
                    :tSPACE,              nil,
                    :tSTRING_END,     nil)
@@ -1815,7 +1899,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
       token = args.shift
       value = args.shift
       assert @lex.advance, "no more tokens"
-      assert_equal [token, value], [@lex.token, @lex.yacc_value]
+      assert_equal [token, value], [@lex.token, [@lex.yacc_value].flatten.first], input
     end
 
     deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"