ruby_parser 3.14.2 → 3.15.0

data/lib/ruby_parser_extras.rb

@@ -1,4 +1,5 @@
 # encoding: ASCII-8BIT
+# TODO: remove
 
 require "sexp"
 require "ruby_lexer"
@@ -28,7 +29,7 @@ class Sexp
 end
 
 module RubyParserStuff
-  VERSION = "3.14.2"
+  VERSION = "3.15.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_accessor :in_kwarg
@@ -45,6 +46,11 @@ module RubyParserStuff
 
   attr_accessor :canonicalize_conditions
 
+  ##
+  # The last token type returned from #next_token
+
+  attr_accessor :last_token_type
+
   $good20 = []
 
   %w[
@@ -493,6 +499,8 @@ module RubyParserStuff
           str.encode! Encoding::UTF_8
           break
         end
+      rescue ArgumentError # unknown encoding name
+        # do nothing
       rescue Encoding::InvalidByteSequenceError
         # do nothing
       rescue Encoding::UndefinedConversionError
@@ -532,7 +540,7 @@ module RubyParserStuff
     header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
 
     first = header.first || ""
-    encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
+    encoding, str = "utf-8", str.b[3..-1] if first =~ /\A\xEF\xBB\xBF/
 
     encoding = $1.strip if header.find { |s|
       s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
@@ -592,7 +600,9 @@ module RubyParserStuff
     case ttype
     when :str then
       if htype == :str
-        head.last << tail.last
+        a, b = head.last, tail.last
+        b = b.dup.force_encoding a.encoding unless Encoding.compatible?(a, b)
+        a << b
       elsif htype == :dstr and head.size == 2 then
         head.last << tail.last
       else
@@ -696,6 +706,15 @@ module RubyParserStuff
     result
   end
 
+  def new_begin val
+    _, lineno, body, _ = val
+
+    result = body ? s(:begin, body) : s(:nil)
+    result.line lineno
+
+    result
+  end
+
   def new_body val
     body, resbody, elsebody, ensurebody = val
 
@@ -723,7 +742,10 @@ module RubyParserStuff
       result = block_append(result, elsebody)
     end
 
-    result = s(:ensure, result, ensurebody).compact.line result.line if ensurebody
+    if ensurebody
+      lineno = (result || ensurebody).line
+      result = s(:ensure, result, ensurebody).compact.line lineno
+    end
 
     result
   end
@@ -843,14 +865,17 @@ module RubyParserStuff
   end
 
   def new_defs val
-    recv, (name, _line), args, body = val[1], val[4], val[6], val[7]
-    line, _ = val[5]
+    _, recv, _, _, name, (_in_def, line), args, body, _ = val
+
     body ||= s(:nil).line line
 
     args.line line
 
     result = s(:defs, recv, name.to_sym, args)
 
+    # TODO: remove_begin
+    # TODO: reduce_nodes
+
     if body then
       if body.sexp_type == :block then
         result.push(*body.sexp_body)
@@ -875,7 +900,9 @@ module RubyParserStuff
   end
 
   def new_hash val
-    s(:hash, *val[2].values).line(val[1])
+    _, line, assocs = val
+
+    s(:hash).line(line).concat assocs.values
   end
 
   def new_if c, t, f
@@ -1134,6 +1161,7 @@ module RubyParserStuff
   def new_string val
     str, = val
     str.force_encoding("UTF-8")
+    # TODO: remove:
     str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
     result = s(:str, str).line lexer.lineno
     self.lexer.fixup_lineno str.count("\n")
@@ -1228,20 +1256,23 @@ module RubyParserStuff
     result
   end
 
-  def new_xstring str
-    if str then
-      case str.sexp_type
+  def new_xstring val
+    _, node = val
+
+    node ||= s(:str, "").line lexer.lineno
+
+    if node then
+      case node.sexp_type
       when :str
-        str.sexp_type = :xstr
+        node.sexp_type = :xstr
       when :dstr
-        str.sexp_type = :dxstr
+        node.sexp_type = :dxstr
       else
-        str = s(:dxstr, "", str).line str.line
+        node = s(:dxstr, "", node).line node.line
       end
-      str
-    else
-      s(:xstr, "")
     end
+
+    node
   end
 
   def new_yield args = nil
@@ -1262,6 +1293,7 @@ module RubyParserStuff
     token = self.lexer.next_token
 
     if token and token.first != RubyLexer::EOF then
+      self.last_token_type = token
       return token
     else
       return [false, false]
@@ -1320,6 +1352,7 @@ module RubyParserStuff
     self.in_single = 0
     self.env.reset
     self.comments.clear
+    self.last_token_type = nil
   end
 
   def ret_args node

data/test/test_ruby_lexer.rb

@@ -1,7 +1,3 @@
-# encoding: US-ASCII
-
-# TODO: work this out
-
 require "minitest/autorun"
 require "ruby_lexer"
 require "ruby_parser"
@@ -84,10 +80,12 @@ class TestRubyLexer < Minitest::Test
       assert_in_epsilon value, act_value, 0.001, msg
     when NilClass then
       assert_nil act_value, msg
+    when String then
+      assert_equal value, act_value.b.force_encoding(value.encoding), msg
     else
       assert_equal value, act_value, msg
     end
-    assert_equal state, @lex.lex_state,  msg if state
+    assert_match state, @lex.lex_state,  msg if state
     assert_equal paren, @lex.paren_nest, msg if paren
     assert_equal brace, @lex.brace_nest, msg if brace
   end
@@ -98,7 +96,8 @@ class TestRubyLexer < Minitest::Test
 
   def assert_read_escape expected, input
     @lex.ss.string = input.dup
-    assert_equal expected, @lex.read_escape.b, input
+    enc = expected.encoding
+    assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
   end
 
   def assert_read_escape_bad input # TODO: rename refute_read_escape
@@ -146,7 +145,7 @@ class TestRubyLexer < Minitest::Test
     yield
 
     lexer.lex_state = EXPR_ENDARG
-    assert_next_lexeme :tSTRING_DEND, "}", EXPR_END, 0
+    assert_next_lexeme :tSTRING_DEND, "}", EXPR_END|EXPR_ENDARG, 0
 
     lexer.lex_strterm = lex_strterm
     lexer.lex_state   = EXPR_BEG
@@ -720,7 +719,7 @@ class TestRubyLexer < Minitest::Test
 
     assert_lex3("do 42 end",
                 nil,
-                :kDO_BLOCK, "do",  EXPR_BEG,
+                :kDO,       "do",  EXPR_BEG,
                 :tINTEGER,  42,    EXPR_NUM,
                 :kEND,      "end", EXPR_END)
   end
@@ -926,6 +925,14 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("$1234", nil, :tGVAR, "$1234", EXPR_END)
   end
 
+  def test_yylex_global_I_have_no_words
+    assert_lex3("$x\xE2\x80\x8B = 42", # zero width space?!?!?
+                nil,
+                :tGVAR, "$x\xE2\x80\x8B", EXPR_END,
+                :tEQL,  "=", EXPR_BEG,
+                :tINTEGER, 42, EXPR_NUM)
+  end
+
   def test_yylex_global_other
     assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
                 nil,
@@ -1109,11 +1116,11 @@ class TestRubyLexer < Minitest::Test
                 :tEQL,            "=",     EXPR_BEG,
                 :tSTRING_BEG,     "\"",    EXPR_BEG,
                 :tSTRING_CONTENT, "#x a ", EXPR_BEG,
-                :tSTRING_DVAR,    "\#@",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "@a b ", EXPR_BEG, # HUH?
-                :tSTRING_DVAR,    "\#$",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "$b c ", EXPR_BEG, # HUH?
-                :tSTRING_DVAR,    "\#@",   EXPR_BEG,
+                :tSTRING_DVAR,    "#",     EXPR_BEG,
                 :tSTRING_CONTENT, "@@d ",  EXPR_BEG, # HUH?
                 :tSTRING_DBEG,    "\#{",   EXPR_BEG,
                 :tSTRING_CONTENT, "3} \n", EXPR_BEG,
@@ -2099,6 +2106,10 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
   end
 
+  def test_yylex_question_control_escape
+    assert_lex3('?\C-\]', nil, :tSTRING, ?\C-\], EXPR_END)
+  end
+
   def test_yylex_question_ws
     assert_lex3("? ",  nil, :tEH, "?", EXPR_BEG)
     assert_lex3("?\n", nil, :tEH, "?", EXPR_BEG)
@@ -2549,11 +2560,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_string_double_escape_c_backslash
-    assert_lex3("\"\\c\\\"",
-                nil,
-                :tSTRING_BEG,     "\"",   EXPR_BEG,
-                :tSTRING_CONTENT, "\034", EXPR_BEG,
-                :tSTRING_END,     "\"",   EXPR_LIT)
+    refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
   end
 
   def test_yylex_string_double_escape_c_escape
@@ -2847,6 +2854,15 @@ class TestRubyLexer < Minitest::Test
                 :tSTRING_END,     '"')
   end
 
+  def test_yylex_string_utf8_bad_encoding_with_escapes
+    str = "\"\\xBADπ\""
+    exp = "\xBADπ".b
+
+    assert_lex(str,
+               s(:str, exp),
+               :tSTRING, exp, EXPR_END)
+  end
+
   def test_yylex_string_utf8_complex_trailing_hex
     chr = [0x3024].pack("U")
     str = "#{chr}abz"

data/test/test_ruby_parser.rb

@@ -185,6 +185,57 @@ module TestRubyParserShared
     assert_syntax_error rb, "else without rescue is useless"
   end
 
+  def test_begin_ensure_no_bodies
+    rb = "begin\nensure\nend"
+    pt = s(:ensure, s(:nil).line(2)).line(2)
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_ensure_no_bodies
+    rb = "begin\nrescue\nensure\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:resbody, s(:array).line(2),
+               nil).line(2)
+            ).line(2),
+           s(:nil).line(3)
+          ).line(2)
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_else_ensure_bodies
+    rb = "begin\n  1\nrescue\n  2\nelse\n  3\nensure\n  4\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:lit, 1).line(2),
+             s(:resbody, s(:array).line(3),
+               s(:lit, 2).line(4)).line(3),
+             s(:lit, 3).line(6)).line(2),
+           s(:lit, 4).line(8)).line(2)
+
+    s(:ensure, s(:rescue, s(:resbody, s(:array), nil)), s(:nil))
+
+    assert_parse rb, pt
+  end
+
+  def test_begin_rescue_else_ensure_no_bodies
+    rb = "begin\n\nrescue\n\nelse\n\nensure\n\nend"
+    pt = s(:ensure,
+           s(:rescue,
+             s(:resbody, s(:array).line(3),
+               # TODO: s(:nil)
+               nil
+              ).line(3),
+            ).line(3),
+          s(:nil).line(7)).line(3)
+
+    s(:ensure, s(:rescue, s(:resbody, s(:array), nil)), s(:nil))
+
+    assert_parse rb, pt
+  end
+
   def test_block_append
     head = s(:args).line 1
     tail = s(:zsuper).line 2
@@ -290,7 +341,7 @@ module TestRubyParserShared
   def test_bug190
     skip "not ready for this yet"
 
-    rb = %{%r'\\''}
+    rb = %{%r'\\\''} # stupid emacs
 
     assert_parse rb, :FUCK
     assert_syntax_error rb, "FUCK"
@@ -872,6 +923,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_heredoc_with_extra_carriage_horrible_mix?
+    rb = "<<'eot'\r\nbody\r\neot\n"
+    pt = s(:str, "body\r\n")
+
+    assert_parse rb, pt
+  end
+
   def test_heredoc_with_interpolation_and_carriage_return_escapes
     rb = "<<EOS\nfoo\\r\#@bar\nEOS\n"
     pt = s(:dstr, "foo\r", s(:evstr, s(:ivar, :@bar)), s(:str, "\n"))
@@ -900,6 +958,13 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
+  def test_heredoc_with_not_global_interpolation
+    rb = "<<-HEREDOC\n#${\nHEREDOC"
+    pt = s(:str, "\#${\n")
+
+    assert_parse rb, pt
+  end
+
   def test_i_fucking_hate_line_numbers
     rb = <<-END.gsub(/^ {6}/, "")
       if true
@@ -1252,6 +1317,20 @@ module TestRubyParserShared
     end
   end
 
+  def test_magic_encoding_comment__bad
+    rb = "#encoding: bunk\n0"
+    pt = s(:lit, 0)
+
+    assert_parse rb, pt
+  end
+
+  def test_utf8_bom
+    rb = "\xEF\xBB\xBF#!/usr/bin/env ruby -w\np 0\n"
+    pt = s(:call, nil, :p, s(:lit, 0))
+
+    assert_parse rb, pt
+  end
+
   def test_masgn_arg_colon_arg
     rb = "a, b::c = d"
     pt = s(:masgn,
@@ -1383,13 +1462,43 @@ module TestRubyParserShared
     assert_parse rb, pt
   end
 
-  def test_op_asgn_primary_colon_identifier
+  def test_op_asgn_primary_colon_identifier1
     rb = "A::b += 1"
     pt = s(:op_asgn, s(:const, :A), s(:lit, 1), :b, :+) # TODO: check? looks wack
 
     assert_parse rb, pt
   end
 
+  def test_lasgn_middle_splat
+    rb = "a = b, *c, d"
+    pt = s(:lasgn, :a,
+           s(:svalue,
+             s(:array,
+               s(:call, nil, :b),
+               s(:splat, s(:call, nil, :c)),
+               s(:call, nil, :d))))
+
+    assert_parse rb, pt
+  end
+
+  def test_op_asgn_primary_colon_const_command_call
+    rb = "A::B *= c d"
+    pt = s(:op_asgn, s(:const, :A),
+           s(:call, nil, :c, s(:call, nil, :d)),
+           :B, :*)
+
+    assert_parse rb, pt
+  end
+
+  def test_op_asgn_primary_colon_identifier_command_call
+    rb = "A::b *= c d"
+    pt = s(:op_asgn, s(:const, :A),
+           s(:call, nil, :c, s(:call, nil, :d)),
+           :b, :*)
+
+    assert_parse rb, pt
+  end
+
   def test_op_asgn_val_dot_ident_command_call
     rb = "a.b ||= c 1"
     pt = s(:op_asgn, s(:call, nil, :a), s(:call, nil, :c, s(:lit, 1)), :b, :"||")
@@ -3288,6 +3397,76 @@ module TestRubyParserShared20Plus
     assert_parse rb, pt
   end
 
+  def test_call_array_block_call
+    rb = "a [ nil, b do end ]"
+    pt = s(:call, nil, :a,
+           s(:array,
+             s(:nil),
+             s(:iter, s(:call, nil, :b), 0)))
+
+    assert_parse rb, pt
+  end
+
+  def test_block_call_paren_call_block_call
+    rb = "a (b)\nc.d do end"
+    pt = s(:block,
+           s(:call, nil, :a, s(:call, nil, :b)),
+           s(:iter, s(:call, s(:call, nil, :c), :d), 0))
+
+
+    assert_parse rb, pt
+  end
+
+  def test_block_call_defn_call_block_call
+    rb = "a def b(c)\n d\n end\n e.f do end"
+    pt = s(:block,
+           s(:call, nil, :a,
+             s(:defn, :b, s(:args, :c), s(:call, nil, :d))),
+           s(:iter, s(:call, s(:call, nil, :e), :f), 0))
+
+    assert_parse rb, pt
+  end
+
+  def test_call_array_lambda_block_call
+    rb = "a [->() {}] do\nend"
+    pt = s(:iter,
+           s(:call, nil, :a,
+             s(:array, s(:iter, s(:lambda), s(:args)))),
+           0)
+
+    assert_parse rb, pt
+  end
+
+  def test_call_begin_call_block_call
+    rb = "a begin\nb.c do end\nend"
+    pt = s(:call, nil, :a,
+           s(:iter, s(:call, s(:call, nil, :b), :c), 0))
+
+    assert_parse rb, pt
+  end
+
+  def test_messy_op_asgn_lineno
+    rb = "a (B::C *= d e)"
+    pt = s(:call, nil, :a,
+           s(:op_asgn, s(:const, :B),
+             s(:call, nil, :d, s(:call, nil, :e)),
+             :C,
+             :*)).line(1)
+
+    assert_parse rb, pt
+  end
+
+  def test_str_lit_concat_bad_encodings
+    rb = '"\xE3\xD3\x8B\xE3\x83\xBC\x83\xE3\x83\xE3\x82\xB3\xA3\x82\x99" \
+        "\xE3\x83\xB3\xE3\x83\x8F\xE3\x82\x9A\xC3\xBD;foo@bar.com"'.b
+    pt = s(:str, "\xE3\xD3\x8B\xE3\x83\xBC\x83\xE3\x83\xE3\x82\xB3\xA3\x82\x99\xE3\x83\xB3\xE3\x83\x8F\xE3\x82\x9A\xC3\xBD;foo@bar.com".b)
+
+    assert_parse rb, pt
+
+    sexp = processor.parse rb
+    assert_equal Encoding::ASCII_8BIT, sexp.last.encoding
+  end
+
   def test_block_call_dot_op2_cmd_args_do_block
     rb = "a.b c() do d end.e f do |g| h end"
     pt = s(:iter,
@@ -3588,7 +3767,7 @@ module TestRubyParserShared21Plus
 
   def test_bug162__21plus
     rb = %q(<<E\nfoo\nE\rO)
-    emsg = "can't match /E(\n|\\z)/ anywhere in . near line 1: \"\""
+    emsg = "can't match /E(\\r*\\n|\\z)/ anywhere in . near line 1: \"\""
 
     assert_syntax_error rb, emsg
   end
@@ -3738,6 +3917,13 @@ module TestRubyParserShared23Plus
     assert_parse rb, pt
   end
 
+  def test_heredoc__backslash_dos_format
+    rb = "str = <<-XXX\r\nbefore\\\r\nafter\r\nXXX\r\n"
+    pt = s(:lasgn, :str, s(:str, "before\nafter\n"))
+
+    assert_parse rb, pt
+  end
+
   def test_heredoc_squiggly
     rb = "a = <<~\"EOF\"\n  x\n  y\n  z\n  EOF\n\n"
     pt = s(:lasgn, :a, s(:str, "x\ny\nz\n"))
@@ -3946,6 +4132,10 @@ module TestRubyParserShared26Plus
   end
 end
 
+module TestRubyParserShared27Plus
+  include TestRubyParserShared26Plus
+end
+
 class TestRubyParser < Minitest::Test
   def test_cls_version
     assert_equal 23, RubyParser::V23.version
@@ -4242,6 +4432,17 @@ class TestRubyParserV26 < RubyParserTestCase
 
 end
 
+class TestRubyParserV27 < RubyParserTestCase
+  include TestRubyParserShared27Plus
+
+  def setup
+    super
+
+    self.processor = RubyParser::V27.new
+  end
+end
+
+
 RubyParser::VERSIONS.each do |klass|
   v = klass.version
   describe "block args arity #{v}" do