ruby_parser 3.14.0 → 3.16.0

data/test/test_ruby_lexer.rb

@@ -1,7 +1,3 @@
-# encoding: US-ASCII
-
-# TODO: work this out
-
 require "minitest/autorun"
 require "ruby_lexer"
 require "ruby_parser"
@@ -84,10 +80,12 @@ class TestRubyLexer < Minitest::Test
       assert_in_epsilon value, act_value, 0.001, msg
     when NilClass then
       assert_nil act_value, msg
+    when String then
+      assert_equal value, act_value.b.force_encoding(value.encoding), msg
     else
       assert_equal value, act_value, msg
     end
-    assert_equal state, @lex.lex_state,  msg if state
+    assert_match state, @lex.lex_state,  msg if state
     assert_equal paren, @lex.paren_nest, msg if paren
     assert_equal brace, @lex.brace_nest, msg if brace
   end
@@ -98,7 +96,8 @@ class TestRubyLexer < Minitest::Test
 
   def assert_read_escape expected, input
     @lex.ss.string = input.dup
-    assert_equal expected, @lex.read_escape.b, input
+    enc = expected.encoding
+    assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
   end
 
   def assert_read_escape_bad input # TODO: rename refute_read_escape
@@ -146,7 +145,7 @@ class TestRubyLexer < Minitest::Test
     yield
 
     lexer.lex_state = EXPR_ENDARG
-    assert_next_lexeme :tSTRING_DEND, "}", EXPR_END, 0
+    assert_next_lexeme :tSTRING_DEND, "}", EXPR_END|EXPR_ENDARG, 0
 
     lexer.lex_strterm = lex_strterm
     lexer.lex_state   = EXPR_BEG
@@ -479,6 +478,22 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
   end
 
+  def test_yylex_bdot2
+    assert_lex3("..42",
+                nil, # TODO: s(:dot2, nil, s(:lit, 42)),
+
+                :tBDOT2,   "..", EXPR_BEG,
+                :tINTEGER, 42,   EXPR_END|EXPR_ENDARG)
+  end
+
+  def test_yylex_bdot3
+    assert_lex3("...42",
+                nil, # TODO: s(:dot2, nil, s(:lit, 42)),
+
+                :tBDOT3,   "...", EXPR_BEG,
+                :tINTEGER, 42,    EXPR_END|EXPR_ENDARG)
+  end
+
   def test_yylex_block_bug_1
     assert_lex3("a do end",
                 s(:iter, s(:call, nil, :a), 0),
@@ -720,7 +735,7 @@ class TestRubyLexer < Minitest::Test
 
     assert_lex3("do 42 end",
                 nil,
-                :kDO_BLOCK, "do",  EXPR_BEG,
+                :kDO,       "do",  EXPR_BEG,
                 :tINTEGER,  42,    EXPR_NUM,
                 :kEND,      "end", EXPR_END)
   end
@@ -750,10 +765,26 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_dot2
+    assert_lex3("1..2",
+                s(:lit, 1..2),
+
+                :tINTEGER, 1,    EXPR_END|EXPR_ENDARG,
+                :tDOT2,    "..", EXPR_BEG,
+                :tINTEGER, 2,    EXPR_END|EXPR_ENDARG)
+
+    self.lex_state = EXPR_END|EXPR_ENDARG
     assert_lex3("..", nil, :tDOT2, "..", EXPR_BEG)
   end
 
   def test_yylex_dot3
+    assert_lex3("1...2",
+                s(:lit, 1...2),
+
+                :tINTEGER, 1,     EXPR_END|EXPR_ENDARG,
+                :tDOT3,    "...", EXPR_BEG,
+                :tINTEGER, 2,     EXPR_END|EXPR_ENDARG)
+
+    self.lex_state = EXPR_END|EXPR_ENDARG
     assert_lex3("...", nil, :tDOT3, "...", EXPR_BEG)
   end
 
@@ -926,6 +957,14 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("$1234", nil, :tGVAR, "$1234", EXPR_END)
   end
 
+  def test_yylex_global_I_have_no_words
+    assert_lex3("$x\xE2\x80\x8B = 42", # zero width space?!?!?
+                nil,
+                :tGVAR, "$x\xE2\x80\x8B", EXPR_END,
+                :tEQL,  "=", EXPR_BEG,
+                :tINTEGER, 42, EXPR_NUM)
+  end
+
   def test_yylex_global_other
     assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
                 nil,
@@ -1109,11 +1148,11 @@ class TestRubyLexer < Minitest::Test
                 :tEQL,            "=",     EXPR_BEG,
                 :tSTRING_BEG,     "\"",    EXPR_BEG,
                 :tSTRING_CONTENT, "#x a ", EXPR_BEG,
-                :tSTRING_DVAR,    "\#@",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "@a b ", EXPR_BEG, # HUH?
-                :tSTRING_DVAR,    "\#$",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "$b c ", EXPR_BEG, # HUH?
-                :tSTRING_DVAR,    "\#@",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "@@d ",  EXPR_BEG, # HUH?
                 :tSTRING_DBEG,    "\#{",   EXPR_BEG,
                 :tSTRING_CONTENT, "3} \n", EXPR_BEG,
@@ -2099,6 +2138,10 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
   end
 
+  def test_yylex_question_control_escape
+    assert_lex3('?\C-\]', nil, :tSTRING, ?\C-\], EXPR_END)
+  end
+
   def test_yylex_question_ws
     assert_lex3("? ",  nil, :tEH, "?", EXPR_BEG)
     assert_lex3("?\n", nil, :tEH, "?", EXPR_BEG)
@@ -2549,11 +2592,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_string_double_escape_c_backslash
-    assert_lex3("\"\\c\\\"",
-                nil,
-                :tSTRING_BEG,     "\"",   EXPR_BEG,
-                :tSTRING_CONTENT, "\034", EXPR_BEG,
-                :tSTRING_END,     "\"",   EXPR_LIT)
+    refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
   end
 
   def test_yylex_string_double_escape_c_escape
@@ -2657,6 +2696,22 @@ class TestRubyLexer < Minitest::Test
                 :tSTRING_END,     "]",     EXPR_LIT)
   end
 
+  def test_yylex_string_pct_Q_null_wtf?
+    assert_lex3("%Q\0s1 s2\0",
+                nil,
+                :tSTRING_BEG,     "%Q\0",  EXPR_BEG,
+                :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
+                :tSTRING_END,     "\0",    EXPR_LIT)
+  end
+
+  def test_yylex_string_pct_Q_bang
+    assert_lex3("%Q!s1 s2!",
+                nil,
+                :tSTRING_BEG,     "%Q\0",  EXPR_BEG,
+                :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
+                :tSTRING_END,     "!",     EXPR_LIT)
+  end
+
   def test_yylex_string_pct_W
     assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
                 nil,
@@ -2831,6 +2886,15 @@ class TestRubyLexer < Minitest::Test
                 :tSTRING_END,     '"')
   end
 
+  def test_yylex_string_utf8_bad_encoding_with_escapes
+    str = "\"\\xBADπ\""
+    exp = "\xBADπ".b
+
+    assert_lex(str,
+               s(:str, exp),
+               :tSTRING, exp, EXPR_END)
+  end
+
   def test_yylex_string_utf8_complex_trailing_hex
     chr = [0x3024].pack("U")
     str = "#{chr}abz"

data/test/test_ruby_parser.rb

@@ -162,6 +162,16 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_backticks_interpolation_line
+    rb = 'x `#{y}`'
+    pt = s(:call, nil, :x,
+           s(:dxstr, "",
+             s(:evstr,
+               s(:call, nil, :y).line(1)).line(1))).line(1)
+
+    assert_parse rb, pt
+  end
+
   def test_bang_eq
     rb = "1 != 2"
     pt = s(:not, s(:call, s(:lit, 1), :"==", s(:lit, 2)))
@@ -175,6 +185,57 @@ module TestRubyParserShared
     assert_syntax_error rb, "else without rescue is useless"
   end
 
+  def test_begin_ensure_no_bodies
+    rb = "begin\nensure\nend"
+    pt = s(:ensure, s(:nil).line(2)).line(2)
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_ensure_no_bodies
+    rb = "begin\nrescue\nensure\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:resbody, s(:array).line(2),
+               nil).line(2)
+            ).line(2),
+           s(:nil).line(3)
+          ).line(2)
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_else_ensure_bodies
+    rb = "begin\n  1\nrescue\n  2\nelse\n  3\nensure\n  4\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:lit, 1).line(2),
+             s(:resbody, s(:array).line(3),
+               s(:lit, 2).line(4)).line(3),
+             s(:lit, 3).line(6)).line(2),
+           s(:lit, 4).line(8)).line(2)
+
+    s(:ensure, s(:rescue, s(:resbody, s(:array), nil)), s(:nil))
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_else_ensure_no_bodies
+    rb = "begin\n\nrescue\n\nelse\n\nensure\n\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:resbody, s(:array).line(3),
+               # TODO: s(:nil)
+               nil
+              ).line(3),
+            ).line(3),
+          s(:nil).line(7)).line(3)
+
+    s(:ensure, s(:rescue, s(:resbody, s(:array), nil)), s(:nil))
+
+    assert_parse rb, pt
+  end
+
   def test_block_append
     head = s(:args).line 1
     tail = s(:zsuper).line 2
@@ -280,7 +341,7 @@ module TestRubyParserShared
   def test_bug190
     skip "not ready for this yet"
 
-    rb = %{%r'\\''}
+    rb = %{%r'\\\''} # stupid emacs
 
     assert_parse rb, :FUCK
     assert_syntax_error rb, "FUCK"
@@ -862,6 +923,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_heredoc_with_extra_carriage_horrible_mix?
+    rb = "<<'eot'\r\nbody\r\neot\n"
+    pt = s(:str, "body\r\n")
+
+    assert_parse rb, pt
+  end
+
   def test_heredoc_with_interpolation_and_carriage_return_escapes
     rb = "<<EOS\nfoo\\r\#@bar\nEOS\n"
     pt = s(:dstr, "foo\r", s(:evstr, s(:ivar, :@bar)), s(:str, "\n"))
@@ -890,6 +958,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_heredoc_with_not_global_interpolation
+    rb = "<<-HEREDOC\n#${\nHEREDOC"
+    pt = s(:str, "\#${\n")
+
+    assert_parse rb, pt
+  end
+
   def test_i_fucking_hate_line_numbers
     rb = <<-END.gsub(/^ {6}/, "")
       if true
@@ -1242,6 +1317,20 @@ module TestRubyParserShared
     end
   end
 
+  def test_magic_encoding_comment__bad
+    rb = "#encoding: bunk\n0"
+    pt = s(:lit, 0)
+
+    assert_parse rb, pt
+  end
+
+  def test_utf8_bom
+    rb = "\xEF\xBB\xBF#!/usr/bin/env ruby -w\np 0\n"
+    pt = s(:call, nil, :p, s(:lit, 0))
+
+    assert_parse rb, pt
+  end
+
   def test_masgn_arg_colon_arg
     rb = "a, b::c = d"
     pt = s(:masgn,
@@ -1373,13 +1462,43 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
-  def test_op_asgn_primary_colon_identifier
+  def test_op_asgn_primary_colon_identifier1
     rb = "A::b += 1"
     pt = s(:op_asgn, s(:const, :A), s(:lit, 1), :b, :+) # TODO: check? looks wack
 
     assert_parse rb, pt
   end
 
+  def test_lasgn_middle_splat
+    rb = "a = b, *c, d"
+    pt = s(:lasgn, :a,
+           s(:svalue,
+             s(:array,
+               s(:call, nil, :b),
+               s(:splat, s(:call, nil, :c)),
+               s(:call, nil, :d))))
+
+    assert_parse rb, pt
+  end
+
+  def test_op_asgn_primary_colon_const_command_call
+    rb = "A::B *= c d"
+    pt = s(:op_asgn, s(:const, :A),
+           s(:call, nil, :c, s(:call, nil, :d)),
+           :B, :*)
+
+    assert_parse rb, pt
+  end
+
+  def test_op_asgn_primary_colon_identifier_command_call
+    rb = "A::b *= c d"
+    pt = s(:op_asgn, s(:const, :A),
+           s(:call, nil, :c, s(:call, nil, :d)),
+           :b, :*)
+
+    assert_parse rb, pt
+  end
+
   def test_op_asgn_val_dot_ident_command_call
     rb = "a.b ||= c 1"
     pt = s(:op_asgn, s(:call, nil, :a), s(:call, nil, :c, s(:lit, 1)), :b, :"||")
@@ -1514,6 +1633,13 @@ module TestRubyParserShared
     assert_equal 2, c.line,   "call should have line number"
   end
 
+  def test_parse_line_defn_no_parens_args
+    rb = "def f a\nend"
+    pt = s(:defn, :f, s(:args, :a).line(1), s(:nil).line(2)).line(1)
+
+    assert_parse_line rb, pt, 1
+  end
+
   def test_parse_line_defn_complex
     rb = "def x(y)\n  p(y)\n  y *= 2\n  return y;\nend" # TODO: remove () & ;
     pt = s(:defn, :x, s(:args, :y),
@@ -1530,7 +1656,7 @@ module TestRubyParserShared
   end
 
   def test_parse_line_defn_no_parens
-    pt = s(:defn, :f, s(:args), s(:nil))
+    pt = s(:defn, :f, s(:args).line(1), s(:nil)).line(1)
 
     rb = "def f\nend"
     assert_parse_line rb, pt, 1
@@ -2954,6 +3080,8 @@ module TestRubyParserShared19Plus
   end
 
   def test_motherfuckin_leading_dots
+    skip if processor.class.version >= 27
+
     rb = "a\n.b"
     pt = s(:call, s(:call, nil, :a), :b)
 
@@ -2961,6 +3089,8 @@ module TestRubyParserShared19Plus
   end
 
   def test_motherfuckin_leading_dots2
+    skip if processor.class.version >= 27
+
     rb = "a\n..b"
 
     assert_parse_error rb, '(string):2 :: parse error on value ".." (tDOT2)'
@@ -3271,6 +3401,76 @@ module TestRubyParserShared20Plus
     assert_parse rb, pt
   end
 
+  def test_call_array_block_call
+    rb = "a [ nil, b do end ]"
+    pt = s(:call, nil, :a,
+           s(:array,
+             s(:nil),
+             s(:iter, s(:call, nil, :b), 0)))
+
+    assert_parse rb, pt
+  end
+
+  def test_block_call_paren_call_block_call
+    rb = "a (b)\nc.d do end"
+    pt = s(:block,
+           s(:call, nil, :a, s(:call, nil, :b)),
+           s(:iter, s(:call, s(:call, nil, :c), :d), 0))
+
+
+    assert_parse rb, pt
+  end
+
+  def test_block_call_defn_call_block_call
+    rb = "a def b(c)\n d\n end\n e.f do end"
+    pt = s(:block,
+           s(:call, nil, :a,
+             s(:defn, :b, s(:args, :c), s(:call, nil, :d))),
+           s(:iter, s(:call, s(:call, nil, :e), :f), 0))
+
+    assert_parse rb, pt
+  end
+
+  def test_call_array_lambda_block_call
+    rb = "a [->() {}] do\nend"
+    pt = s(:iter,
+           s(:call, nil, :a,
+             s(:array, s(:iter, s(:lambda), s(:args)))),
+           0)
+
+    assert_parse rb, pt
+  end
+
+  def test_call_begin_call_block_call
+    rb = "a begin\nb.c do end\nend"
+    pt = s(:call, nil, :a,
+           s(:iter, s(:call, s(:call, nil, :b), :c), 0))
+
+    assert_parse rb, pt
+  end
+
+  def test_messy_op_asgn_lineno
+    rb = "a (B::C *= d e)"
+    pt = s(:call, nil, :a,
+           s(:op_asgn, s(:const, :B),
+             s(:call, nil, :d, s(:call, nil, :e)),
+             :C,
+             :*)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_str_lit_concat_bad_encodings
+    rb = '"\xE3\xD3\x8B\xE3\x83\xBC\x83\xE3\x83\xE3\x82\xB3\xA3\x82\x99" \
+        "\xE3\x83\xB3\xE3\x83\x8F\xE3\x82\x9A\xC3\xBD;foo@bar.com"'.b
+    pt = s(:str, "\xE3\xD3\x8B\xE3\x83\xBC\x83\xE3\x83\xE3\x82\xB3\xA3\x82\x99\xE3\x83\xB3\xE3\x83\x8F\xE3\x82\x9A\xC3\xBD;foo@bar.com".b)
+
+    assert_parse rb, pt
+
+    sexp = processor.parse rb
+    assert_equal Encoding::ASCII_8BIT, sexp.last.encoding
+  end
+
   def test_block_call_dot_op2_cmd_args_do_block
     rb = "a.b c() do d end.e f do |g| h end"
     pt = s(:iter,
@@ -3571,7 +3771,7 @@ module TestRubyParserShared21Plus
 
   def test_bug162__21plus
     rb = %q(<<E\nfoo\nE\rO)
-    emsg = "can't match /E(\n|\\z)/ anywhere in . near line 1: \"\""
+    emsg = "can't match /E(\\r*\\n|\\z)/ anywhere in . near line 1: \"\""
 
     assert_syntax_error rb, emsg
   end
@@ -3721,6 +3921,13 @@ module TestRubyParserShared23Plus
     assert_parse rb, pt
   end
 
+  def test_heredoc__backslash_dos_format
+    rb = "str = <<-XXX\r\nbefore\\\r\nafter\r\nXXX\r\n"
+    pt = s(:lasgn, :str, s(:str, "before\nafter\n"))
+
+    assert_parse rb, pt
+  end
+
   def test_heredoc_squiggly
     rb = "a = <<~\"EOF\"\n  x\n  y\n  z\n  EOF\n\n"
     pt = s(:lasgn, :a, s(:str, "x\ny\nz\n"))
@@ -3929,6 +4136,14 @@ module TestRubyParserShared26Plus
   end
 end
 
+module TestRubyParserShared27Plus
+  include TestRubyParserShared26Plus
+end
+
+module TestRubyParserShared30Plus
+  include TestRubyParserShared27Plus
+end
+
 class TestRubyParser < Minitest::Test
   def test_cls_version
     assert_equal 23, RubyParser::V23.version
@@ -4225,6 +4440,46 @@ class TestRubyParserV26 < RubyParserTestCase
 
 end
 
+class TestRubyParserV27 < RubyParserTestCase
+  include TestRubyParserShared27Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V27.new
+  end
+
+  def test_bdot2
+    rb = "..10\n; ..a\n; c"
+    pt = s(:block,
+           s(:dot2, nil, s(:lit, 10).line(1)).line(1),
+           s(:dot2, nil, s(:call, nil, :a).line(2)).line(2),
+           s(:call, nil, :c).line(3)).line(1)
+
+    assert_parse_line rb, pt, 1
+  end
+
+  def test_bdot3
+    rb = "...10\n; ...a\n; c"
+    pt = s(:block,
+           s(:dot3, nil, s(:lit, 10).line(1)).line(1),
+           s(:dot3, nil, s(:call, nil, :a).line(2)).line(2),
+           s(:call, nil, :c).line(3)).line(1)
+
+    assert_parse_line rb, pt, 1
+  end
+end
+
+class TestRubyParserV30 < RubyParserTestCase
+  include TestRubyParserShared30Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V30.new
+  end
+end
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do