ruby_parser 3.12.0 → 3.13.0

data/lib/ruby_parser_extras.rb

@@ -7,9 +7,10 @@ require "rp_extensions"
 require "rp_stringscanner"
 
 module RubyParserStuff
-  VERSION = "3.12.0"
+  VERSION = "3.13.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
+  attr_accessor :in_kwarg
   attr_reader :env, :comments
 
   $good20 = []
@@ -90,23 +91,6 @@ module RubyParserStuff
     result
   end
 
-  def block_var18 ary, splat, block
-    ary ||= s(:array)
-
-    if splat then
-      splat = splat[1] unless Symbol === splat
-      ary << "*#{splat}".to_sym
-    end
-
-    ary << "&#{block[1]}".to_sym if block
-
-    if ary.length > 2 or ary.splat then # HACK
-      s(:masgn, *ary.sexp_body)
-    else
-      ary.last
-    end
-  end
-
   def array_to_hash array
     case array.sexp_type
     when :kwsplat then
@@ -152,8 +136,10 @@ module RubyParserStuff
         when :block_arg then
           result << :"&#{arg.last}"
         when :shadow then
+          name = arg.last
+          self.env[name] = :lvar
           if Sexp === result.last and result.last.sexp_type == :shadow then
-            result.last << arg.last
+            result.last << name
           else
             result << arg
           end
@@ -288,9 +274,6 @@ module RubyParserStuff
     return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
   end
 
-  # TODO: remove in 4.0 or 2018-01, whichever is first
-  deprecate :get_match_node, :new_match
-
   def gettable(id)
     lineno = id.lineno if id.respond_to? :lineno
     id = id.to_sym if String === id
@@ -334,10 +317,12 @@ module RubyParserStuff
   def initialize(options = {})
     super()
 
-    v = self.class.name[/1[89]|2[01]/]
+    v = self.class.name[/2\d/]
+    raise "Bad Class name #{self.class}" unless v
 
     self.lexer = RubyLexer.new v && v.to_i
     self.lexer.parser = self
+    self.in_kwarg = false
 
     @env = RubyParserStuff::Environment.new
     @comments = []
@@ -431,9 +416,6 @@ module RubyParserStuff
     return s(type, left, right)
   end
 
-  # TODO: remove in 4.0 or 2018-01, whichever is first
-  deprecate :logop, :logical_op
-
   def new_aref val
     val[2] ||= s(:arglist)
     val[2].sexp_type = :arglist if val[2].sexp_type == :array # REFACTOR
@@ -472,6 +454,10 @@ module RubyParserStuff
     result
   end
 
+  def new_brace_body args, body, lineno
+    new_iter(nil, args, body).line(lineno)
+  end
+
   def argl x
     x = s(:arglist, x) if x and x.sexp_type == :array
     x
@@ -580,7 +566,7 @@ module RubyParserStuff
   end
 
   def new_defn val
-    (_, line), name, _, args, body, * = val
+    (_, line), (name, _), _, args, body, * = val
     body ||= s(:nil)
 
     result = s(:defn, name.to_sym, args)
@@ -601,7 +587,7 @@ module RubyParserStuff
   end
 
   def new_defs val
-    recv, name, args, body = val[1], val[4], val[6], val[7]
+    recv, (name, _line), args, body = val[1], val[4], val[6], val[7]
     body ||= s(:nil)
 
     result = s(:defs, recv, name.to_sym, args)
@@ -619,6 +605,10 @@ module RubyParserStuff
     result
   end
 
+  def new_do_body args, body, lineno
+    new_iter(nil, args, body).line(lineno)
+  end
+
   def new_for expr, var, body
     result = s(:for, expr, var).line(var.line)
     result << body if body
@@ -743,9 +733,6 @@ module RubyParserStuff
       }[c]
       raise "unknown regexp option: #{c}" unless v
       o += v
-
-      # encoding options are ignored on 1.9+
-      k = c if c =~ /[esu]/ if RUBY_VERSION < "1.9"
     end
 
     case node.sexp_type
@@ -783,6 +770,10 @@ module RubyParserStuff
     node
   end
 
+  def new_rescue body, resbody
+    s(:rescue, body, resbody)
+  end
+
   def new_resbody cond, body
     if body && body.sexp_type == :block then
       body.shift # remove block and splat it in directly
@@ -813,7 +804,7 @@ module RubyParserStuff
 
   def new_string val
     str = val[0]
-    str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
     result = s(:str, str)
     self.lexer.fixup_lineno str.count("\n")
     result
@@ -821,7 +812,7 @@ module RubyParserStuff
 
   def new_qword_list_entry val
     str = val[1]
-    str.force_encoding("ASCII-8BIT") unless str.valid_encoding? unless RUBY_VERSION < "1.9"
+    str.force_encoding("ASCII-8BIT") unless str.valid_encoding?
     result = s(:str, str)
     self.lexer.fixup_lineno
     result
@@ -875,7 +866,6 @@ module RubyParserStuff
     when :str then
       sym = s(:lit, sym.last.to_sym)
     else
-      debug20 24
       sym = s(:dsym, "", sym || s(:str, ""))
     end
     self.lexer.fixup_lineno
@@ -986,9 +976,6 @@ module RubyParserStuff
     lhs
   end
 
-  # TODO: remove in 4.0 or 2018-01, whichever is first
-  deprecate :node_assign, :new_assign
-
   ##
   # Returns a UTF-8 encoded string after processing BOMs and magic
   # encoding comments.
@@ -1145,7 +1132,7 @@ module RubyParserStuff
     result
   end
 
-  def value_expr oldnode # HACK
+  def value_expr oldnode # HACK: much more to do
     node = remove_begin oldnode
     node.line = oldnode.line if oldnode
     node[2] = value_expr node[2] if node and node.sexp_type == :if
@@ -1193,82 +1180,71 @@ module RubyParserStuff
     ##
     # :stopdoc:
     #
-    # :expr_beg    = ignore newline, +/- is a sign.
-    # :expr_end    = newline significant, +/- is a operator.
-    # :expr_arg    = newline significant, +/- is a operator.
-    # :expr_cmdarg = newline significant, +/- is a operator.
-    # :expr_endarg = newline significant, +/- is a operator.
-    # :expr_mid    = newline significant, +/- is a operator.
-    # :expr_fname  = ignore newline, no reserved words.
-    # :expr_dot    = right after . or ::, no reserved words.
-    # :expr_class  = immediate after class, no here document.
+    # :expr_beg     = ignore newline, +/- is a sign.
+    # :expr_end     = newline significant, +/- is an operator.
+    # :expr_endarg  = ditto, and unbound braces.
+    # :expr_endfn   = ditto, and unbound braces.
+    # :expr_arg     = newline significant, +/- is an operator.
+    # :expr_cmdarg  = ditto
+    # :expr_mid     = ditto
+    # :expr_fname   = ignore newline, no reserved words.
+    # :expr_dot     = right after . or ::, no reserved words.
+    # :expr_class   = immediate after class, no here document.
+    # :expr_label   = flag bit, label is allowed.
+    # :expr_labeled = flag bit, just after a label.
+    # :expr_fitem   = symbol literal as FNAME.
+    # :expr_value   = :expr_beg -- work to remove. Need multi-state support.
 
     wordlist = [
-                ["end",      [:kEND,      :kEND        ], :expr_end   ],
-                ["else",     [:kELSE,     :kELSE       ], :expr_beg   ],
+                ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
+                ["and",      [:kAND,      :kAND        ], :expr_beg   ],
+                ["begin",    [:kBEGIN,    :kBEGIN      ], :expr_beg   ],
+                ["break",    [:kBREAK,    :kBREAK      ], :expr_mid   ],
                 ["case",     [:kCASE,     :kCASE       ], :expr_beg   ],
+                ["class",    [:kCLASS,    :kCLASS      ], :expr_class ],
+                ["def",      [:kDEF,      :kDEF        ], :expr_fname ],
+                ["defined?", [:kDEFINED,  :kDEFINED    ], :expr_arg   ],
+                ["do",       [:kDO,       :kDO         ], :expr_beg   ],
+                ["else",     [:kELSE,     :kELSE       ], :expr_beg   ],
+                ["elsif",    [:kELSIF,    :kELSIF      ], :expr_beg   ],
+                ["end",      [:kEND,      :kEND        ], :expr_end   ],
                 ["ensure",   [:kENSURE,   :kENSURE     ], :expr_beg   ],
+                ["false",    [:kFALSE,    :kFALSE      ], :expr_end   ],
+                ["for",      [:kFOR,      :kFOR        ], :expr_beg   ],
+                ["if",       [:kIF,       :kIF_MOD     ], :expr_beg   ],
+                ["in",       [:kIN,       :kIN         ], :expr_beg   ],
                 ["module",   [:kMODULE,   :kMODULE     ], :expr_beg   ],
-                ["elsif",    [:kELSIF,    :kELSIF      ], :expr_beg   ],
-                ["def",      [:kDEF,      :kDEF        ], :expr_fname ],
+                ["next",     [:kNEXT,     :kNEXT       ], :expr_mid   ],
+                ["nil",      [:kNIL,      :kNIL        ], :expr_end   ],
+                ["not",      [:kNOT,      :kNOT        ], :expr_arg   ],
+                ["or",       [:kOR,       :kOR         ], :expr_beg   ],
+                ["redo",     [:kREDO,     :kREDO       ], :expr_end   ],
                 ["rescue",   [:kRESCUE,   :kRESCUE_MOD ], :expr_mid   ],
-                ["not",      [:kNOT,      :kNOT        ], :expr_beg   ],
-                ["then",     [:kTHEN,     :kTHEN       ], :expr_beg   ],
-                ["yield",    [:kYIELD,    :kYIELD      ], :expr_arg   ],
-                ["for",      [:kFOR,      :kFOR        ], :expr_beg   ],
-                ["self",     [:kSELF,     :kSELF       ], :expr_end   ],
-                ["false",    [:kFALSE,    :kFALSE      ], :expr_end   ],
                 ["retry",    [:kRETRY,    :kRETRY      ], :expr_end   ],
                 ["return",   [:kRETURN,   :kRETURN     ], :expr_mid   ],
-                ["true",     [:kTRUE,     :kTRUE       ], :expr_end   ],
-                ["if",       [:kIF,       :kIF_MOD     ], :expr_beg   ],
-                ["defined?", [:kDEFINED,  :kDEFINED    ], :expr_arg   ],
+                ["self",     [:kSELF,     :kSELF       ], :expr_end   ],
                 ["super",    [:kSUPER,    :kSUPER      ], :expr_arg   ],
+                ["then",     [:kTHEN,     :kTHEN       ], :expr_beg   ],
+                ["true",     [:kTRUE,     :kTRUE       ], :expr_end   ],
                 ["undef",    [:kUNDEF,    :kUNDEF      ], :expr_fname ],
-                ["break",    [:kBREAK,    :kBREAK      ], :expr_mid   ],
-                ["in",       [:kIN,       :kIN         ], :expr_beg   ],
-                ["do",       [:kDO,       :kDO         ], :expr_beg   ],
-                ["nil",      [:kNIL,      :kNIL        ], :expr_end   ],
-                ["until",    [:kUNTIL,    :kUNTIL_MOD  ], :expr_beg   ],
                 ["unless",   [:kUNLESS,   :kUNLESS_MOD ], :expr_beg   ],
-                ["or",       [:kOR,       :kOR         ], :expr_beg   ],
-                ["next",     [:kNEXT,     :kNEXT       ], :expr_mid   ],
+                ["until",    [:kUNTIL,    :kUNTIL_MOD  ], :expr_beg   ],
                 ["when",     [:kWHEN,     :kWHEN       ], :expr_beg   ],
-                ["redo",     [:kREDO,     :kREDO       ], :expr_end   ],
-                ["and",      [:kAND,      :kAND        ], :expr_beg   ],
-                ["begin",    [:kBEGIN,    :kBEGIN      ], :expr_beg   ],
-                ["__LINE__", [:k__LINE__, :k__LINE__   ], :expr_end   ],
-                ["class",    [:kCLASS,    :kCLASS      ], :expr_class ],
-                ["__FILE__", [:k__FILE__, :k__FILE__   ], :expr_end   ],
-                ["END",      [:klEND,     :klEND       ], :expr_end   ],
-                ["BEGIN",    [:klBEGIN,   :klBEGIN     ], :expr_end   ],
                 ["while",    [:kWHILE,    :kWHILE_MOD  ], :expr_beg   ],
-                ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
+                ["yield",    [:kYIELD,    :kYIELD      ], :expr_arg   ],
+                ["BEGIN",    [:klBEGIN,   :klBEGIN     ], :expr_end   ],
+                ["END",      [:klEND,     :klEND       ], :expr_end   ],
+                ["__FILE__", [:k__FILE__, :k__FILE__   ], :expr_end   ],
+                ["__LINE__", [:k__LINE__, :k__LINE__   ], :expr_end   ],
                 ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], :expr_end],
                ].map { |args| KWtable.new(*args) }
 
     # :startdoc:
 
-    WORDLIST18 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
-    WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+    WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
 
-    WORDLIST18.delete "__ENCODING__"
-
-    %w[and case elsif for if in module or unless until when while].each do |k|
-      WORDLIST19[k] = WORDLIST19[k].dup
-      WORDLIST19[k].state = :expr_value
-    end
-    %w[not].each do |k|
-      WORDLIST19[k] = WORDLIST19[k].dup
-      WORDLIST19[k].state = :expr_arg
-    end
-
-    def self.keyword18 str # REFACTOR
-      WORDLIST18[str]
-    end
-
-    def self.keyword19 str
-      WORDLIST19[str]
+    def self.keyword str
+      WORDLIST[str]
     end
   end
 
@@ -1330,7 +1306,7 @@ module RubyParserStuff
 
     def reset
       @stack = [false]
-      warn "#{name}_stack(set): 0" if debug
+      log :reset if debug
     end
 
     def inspect
@@ -1338,41 +1314,47 @@ module RubyParserStuff
     end
 
     def is_in_state
+      log :is_in_state if debug
       @stack.last
     end
 
     def lexpop
-      warn "#{name}_stack.lexpop" if debug
       raise if @stack.size == 0
       a = @stack.pop
       b = @stack.pop
       @stack.push(a || b)
+      log :lexpop if debug
+    end
+
+    def log action
+      c = caller[1]
+      c = caller[2] if c =~ /expr_result/
+      warn "%s_stack.%s: %p at %s" % [name, action, @stack, c.clean_caller]
+      nil
     end
 
     def pop
       r = @stack.pop
-      warn "#{name}_stack.pop" if debug
-      @stack.push false if @stack.size == 0
+      @stack.push false if @stack.empty?
+      log :pop if debug
       r
     end
 
     def push val
       @stack.push val
-      return unless debug
-      c = caller.first
-      c = caller[1] if c =~ /expr_result/
-      warn "#{name}_stack(push): #{val} at line #{c.clean_caller}"
-      nil
+      log :push if debug
     end
 
-    def store
+    def store base = false
       result = @stack.dup
-      @stack.replace [false]
+      @stack.replace [base]
+      log :store if debug
       result
     end
 
     def restore oldstate
       @stack.replace oldstate
+      log :restore if debug
     end
   end
 end

data/test/test_ruby_lexer.rb

@@ -51,10 +51,6 @@ class TestRubyLexer < Minitest::Test
     assert_lex(input, exp_sexp, *args, &block)
   end
 
-  def ruby18
-    RubyParser::V18 === lexer.parser
-  end
-
   def refute_lex input, *args # TODO: re-sort
     args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
 
@@ -141,7 +137,7 @@ class TestRubyLexer < Minitest::Test
     yield
 
     lexer.lex_state = :expr_endarg
-    assert_next_lexeme :tRCURLY,     "}",  :expr_endarg, 0
+    assert_next_lexeme :tSTRING_DEND, "}",  :expr_endarg, 0
 
     lexer.lex_strterm = lex_strterm
     lexer.lex_state   = :expr_beg
@@ -304,44 +300,23 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
   end
 
-  def test_yylex_label__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("{a:",
-                nil,
-                :tLBRACE,     "{", :expr_beg,
-                :tIDENTIFIER, "a", :expr_arg,
-                :tSYMBEG,     ":", :expr_fname)
-  end
-
-  def test_yylex_label_in_params__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("foo(a:",
-                nil,
-                :tIDENTIFIER, "foo", :expr_cmdarg,
-                :tLPAREN2,    "(",   :expr_beg,
-                :tIDENTIFIER, "a",   :expr_cmdarg,
-                :tSYMBEG,     ":",   :expr_fname)
-  end
-
-  def test_yylex_label__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_label__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex3("{a:",
                 nil,
                 :tLBRACE, "{", :expr_beg,
-                :tLABEL,  "a", :expr_labelarg)
+                :tLABEL,  "a", :expr_labeled)
   end
 
-  def test_yylex_label_in_params__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_label_in_params__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex3("foo(a:",
                 nil,
                 :tIDENTIFIER, "foo", :expr_cmdarg,
                 :tLPAREN2,    "(",   :expr_beg,
-                :tLABEL,      "a",   :expr_labelarg)
+                :tLABEL,      "a",   :expr_labeled)
   end
 
   def test_yylex_paren_string_parens_interpolated
@@ -357,13 +332,13 @@ class TestRubyLexer < Minitest::Test
     assert_next_lexeme :tSTRING_DBEG,    nil,  :expr_beg, 0, 0
 
     emulate_string_interpolation do
-      assert_next_lexeme :tIDENTIFIER,   "b",  :expr_arg, 0, 0
+      assert_next_lexeme :tIDENTIFIER,   "b",  :expr_cmdarg, 0, 0
     end
 
     assert_next_lexeme :tSTRING_DBEG,    nil,  :expr_beg, 0, 0
 
     emulate_string_interpolation do
-      assert_next_lexeme :tIDENTIFIER,   "d",  :expr_arg, 0, 0
+      assert_next_lexeme :tIDENTIFIER,   "d",  :expr_cmdarg, 0, 0
     end
 
     assert_next_lexeme :tSTRING_CONTENT, ")",  :expr_beg, 0, 0
@@ -503,7 +478,7 @@ class TestRubyLexer < Minitest::Test
                 :tLAMBDA,     nil,   :expr_endfn,
                 :kDO,         "do",  :expr_beg,
                 :kEND,        "end", :expr_end,
-                :kDO_BLOCK,   "do",  :expr_beg,
+                :kDO,         "do",  :expr_beg,
                 :kEND,        "end", :expr_end)
   end
 
@@ -712,6 +687,39 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
   end
 
+  def test_yylex_block_bug_1
+    assert_lex3("a do end",
+                s(:iter, s(:call, nil, :a), 0),
+
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_block_bug_2
+    assert_lex3("a = 1\na do\nend",
+                s(:block,
+                  s(:lasgn, :a, s(:lit, 1)),
+                  s(:iter, s(:call, nil, :a), 0)),
+
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :tEQL,        "=",   :expr_beg,
+                :tINTEGER,    1,     :expr_end,
+                :tNL,         nil,   :expr_beg,
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_block_bug_3
+    assert_lex3("a { }",
+                s(:iter, s(:call, nil, :a), 0),
+
+                :tIDENTIFIER, "a", :expr_cmdarg, # verified
+                :tLCURLY,     "{", :expr_beg, # TODO: expr_beg|expr_label
+                :tRCURLY,     "}", :expr_endarg)
+  end
+
   def test_yylex_carat
     assert_lex3("^", nil, :tCARET, "^", :expr_beg)
   end
@@ -731,8 +739,8 @@ class TestRubyLexer < Minitest::Test
   def test_yylex_colon2_argh
     assert_lex3("module X::Y\n  c\nend",
                 nil,
-                :kMODULE,     "module", :expr_value,
-                :tCONSTANT,   "X",      :expr_arg,
+                :kMODULE,     "module", :expr_beg,
+                :tCONSTANT,   "X",      :expr_cmdarg,
                 :tCOLON2,     "::",     :expr_dot,
                 :tCONSTANT,   "Y",      :expr_arg,
                 :tNL,         nil,      :expr_beg,
@@ -1369,14 +1377,8 @@ class TestRubyLexer < Minitest::Test
     assert_lex_fname "<=>", :tCMP
   end
 
-  def test_yylex_identifier_def__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
-  end
-
-  def test_yylex_identifier_def__1920
-    setup_lexer_class RubyParser::V19
+  def test_yylex_identifier_def__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
   end
@@ -1412,14 +1414,8 @@ class TestRubyLexer < Minitest::Test
     assert_lex_fname "^", :tCARET
   end
 
-  def test_yylex_identifier_equals_def__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
-  end
-
-  def test_yylex_identifier_equals_def__1920
-    setup_lexer_class RubyParser::V19
+  def test_yylex_identifier_equals_def__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
   end
@@ -1521,26 +1517,14 @@ class TestRubyLexer < Minitest::Test
                 :kIF_MOD, "if", :expr_beg)
   end
 
-  def test_yylex_question_eh_a__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
-  end
-
-  def test_yylex_question_eh_a__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_question_eh_a__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
   end
 
-  def test_yylex_question_eh_escape_M_escape_C__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
-  end
-
-  def test_yylex_question_eh_escape_M_escape_C__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_question_eh_escape_M_escape_C__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
   end
@@ -1724,15 +1708,8 @@ class TestRubyLexer < Minitest::Test
     assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
   end
 
-  def test_yylex_open_bracket_exprarg__18
-    setup_lexer_class RubyParser::V18
-    self.lex_state = :expr_arg
-
-    assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
-  end
-
-  def test_yylex_open_bracket_exprarg__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_open_bracket_exprarg__20
+    setup_lexer_class RubyParser::V20
     self.lex_state = :expr_arg
 
     assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
@@ -1896,18 +1873,6 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
   end
 
-  def test_yylex_question__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
-  end
-
-  def test_yylex_question__19
-    setup_lexer_class RubyParser::V19
-
-    assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
-  end
-
   def test_yylex_question_bad_eos
     refute_lex "?"
   end
@@ -1921,19 +1886,8 @@ class TestRubyLexer < Minitest::Test
     assert_lex3("?\f", nil, :tEH, "?", :expr_value)
   end
 
-  def test_yylex_question_ws_backslashed__18
-    setup_lexer_class RubyParser::V18
-
-    assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
-    assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
-    assert_lex3("?\\t", nil, :tINTEGER,  9, :expr_end)
-    assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
-    assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
-    assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
-  end
-
-  def test_yylex_question_ws_backslashed__19
-    setup_lexer_class RubyParser::V19
+  def test_yylex_question_ws_backslashed__20
+    setup_lexer_class RubyParser::V20
 
     assert_lex3("?\\ ", nil, :tSTRING, " ",  :expr_end)
     assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
@@ -1948,7 +1902,9 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_rcurly
-    assert_lex3("}", nil, :tRCURLY, "}", :expr_endarg)
+    assert_lex("}", nil, :tRCURLY, "}", :expr_endarg, 0, 1) do
+      lexer.brace_nest += 2
+    end
   end
 
   def test_yylex_regexp
@@ -2434,8 +2390,6 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_string_double_pound_dollar_bad
-    skip if ruby18
-
     assert_lex3('"#$%"', nil,
 
                 :tSTRING_BEG,     "\"",  :expr_beg,
@@ -2619,12 +2573,6 @@ class TestRubyLexer < Minitest::Test
     assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
   end
 
-  def test_yylex_symbol_zero_byte__18
-    setup_lexer_class RubyParser::V18
-
-    refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
-  end
-
   def test_yylex_symbol_zero_byte
     assert_lex(":\"symbol\0\"", nil,
                 :tSYMBOL,         "symbol\0", :expr_end)
@@ -2791,7 +2739,7 @@ class TestRubyLexer < Minitest::Test
                s(:hash, s(:lit, :a), s(:lit, 1)),
 
                :tLBRACE, "{", :expr_beg,      0, 1,
-               :tLABEL,  "a", :expr_labelarg, 0, 1,
+               :tLABEL,  "a", :expr_labeled, 0, 1,
                :tINTEGER, 1,  :expr_end,      0, 1,
                :tRCURLY, "}", :expr_endarg,   0, 0)
   end
@@ -2803,7 +2751,7 @@ class TestRubyLexer < Minitest::Test
                s(:hash, s(:lit, :a), s(:lit, 1)),
 
                :tLBRACE, "{", :expr_beg,      0, 1,
-               :tLABEL,  "a", :expr_labelarg, 0, 1,
+               :tLABEL,  "a", :expr_labeled, 0, 1,
                :tINTEGER, 1,  :expr_end,      0, 1,
                :tRCURLY, "}", :expr_endarg,   0, 0)
   end
@@ -2815,7 +2763,7 @@ class TestRubyLexer < Minitest::Test
                s(:hash, s(:lit, :abc), s(:lit, :b)),
 
                :tLBRACE, "{", :expr_beg,      0, 1,
-               :tLABEL,  "abc", :expr_labelarg, 0, 1,
+               :tLABEL,  "abc", :expr_labeled, 0, 1,
                :tSYMBOL, "b",  :expr_end,     0, 1,
                :tRCURLY, "}", :expr_endarg,   0, 0)
   end
@@ -2827,7 +2775,7 @@ class TestRubyLexer < Minitest::Test
                s(:hash, s(:lit, :abc), s(:lit, :b)),
 
                :tLBRACE, "{", :expr_beg,      0, 1,
-               :tLABEL,  "abc", :expr_labelarg, 0, 1,
+               :tLABEL,  "abc", :expr_labeled, 0, 1,
                :tSYMBOL, "b",  :expr_end,     0, 1,
                :tRCURLY, "}", :expr_endarg,   0, 0)
   end
@@ -2839,10 +2787,9 @@ class TestRubyLexer < Minitest::Test
                 nil,
                 :kDEF, "def", :expr_fname,
                 :tIDENTIFIER, "foo", :expr_endfn,
-                :tLABEL, "a",   :expr_labelarg,
+                :tLABEL, "a",   :expr_labeled,
                 :tCOMMA, ",", :expr_beg,
-                :tLABEL, "b",   :expr_labelarg,
-                :tNL, nil, :expr_beg,
+                :tLABEL, "b",   :expr_labeled,
                 :kEND, "end", :expr_end)
   end
 
@@ -2853,7 +2800,7 @@ class TestRubyLexer < Minitest::Test
                nil,
 
                :tLBRACE, "{", :expr_beg,
-               :tLABEL,  "s\tr\i\ng\\foo'bar", :expr_labelarg,
+               :tLABEL,  "s\tr\i\ng\\foo'bar", :expr_labeled,
                :tINTEGER, 1,  :expr_end,
                :tRCURLY, "}", :expr_endarg)
   end
@@ -2865,7 +2812,7 @@ class TestRubyLexer < Minitest::Test
                nil,
 
                :tLBRACE, "{", :expr_beg,
-               :tLABEL,  "s\\tr\\i\\ng\\foo'bar", :expr_labelarg,
+               :tLABEL,  "s\\tr\\i\\ng\\foo'bar", :expr_labeled,
                :tINTEGER, 1,  :expr_end,
                :tRCURLY, "}", :expr_endarg)
   end