ruby_parser-legacy 1.0.0

data/test/ruby_parser/test_legacy.rb

@@ -0,0 +1,8 @@
+require "minitest/autorun"
+require "ruby_parser/legacy"
+
+class TestLegacy < Minitest::Test
+  def test_sanity
+    skip "write tests or I will kneecap you"
+  end
+end

data/test/ruby_parser/test_ruby_lexer.rb

@@ -0,0 +1,2984 @@
+# encoding: US-ASCII
+
+require "minitest/autorun"
+require "ruby_lexer"
+require "ruby_parser"
+
+class TestRubyLexer < Minitest::Test
+  attr_accessor :processor, :lex, :parser_class, :lex_state
+
+  alias :lexer :lex # lets me copy/paste code from parser
+  alias :lexer= :lex=
+
+  def setup
+    self.lex_state = :expr_beg
+    setup_lexer_class RubyParser::V19
+  end
+
+  def setup_lexer input, exp_sexp = nil
+    setup_new_parser
+    lex.ss = RPStringScanner.new(input)
+    lex.lex_state = self.lex_state
+  end
+
+  def setup_new_parser
+    self.processor = parser_class.new
+    self.lex = processor.lexer
+  end
+
+  def setup_lexer_class parser_class
+    self.parser_class = parser_class
+    setup_new_parser
+    setup_lexer "blah blah"
+  end
+
+  def assert_lex input, exp_sexp, *args, &b
+    setup_lexer input
+    assert_parse input, exp_sexp if exp_sexp
+
+    b.call if b
+
+    args.each_slice(5) do |token, value, state, paren, brace|
+      assert_next_lexeme token, value, state, paren, brace
+    end
+
+    refute_lexeme
+  end
+
+  def assert_lex3 input, exp_sexp, *args, &block
+    args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
+
+    assert_lex(input, exp_sexp, *args, &block)
+  end
+
+  def ruby18
+    RubyParser::V18 === lexer.parser
+  end
+
+  def refute_lex input, *args # TODO: re-sort
+    args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
+
+    assert_raises RubyParser::SyntaxError do
+      assert_lex(input, nil, *args)
+    end
+  end
+
+  def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
+    assert_lex3("def #{name} ",
+                nil,
+
+                :kDEF, "def", :expr_fname,
+                type,  name,  end_state)
+  end
+
+  def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
+    adv = @lex.next_token
+
+    assert adv, "no more tokens"
+
+    act_token, act_value = adv
+
+    msg = message {
+      act = [act_token, act_value, @lex.lex_state,
+             @lex.paren_nest, @lex.brace_nest]
+      exp = [token, value, state, paren, brace]
+      "#{exp.inspect} vs #{act.inspect}"
+    }
+
+    act_value = act_value.first if Array === act_value
+
+    assert_equal token, act_token,       msg
+    case value
+    when Float then
+      assert_in_epsilon value, act_value, 0.001, msg
+    when NilClass then
+      assert_nil act_value, msg
+    else
+      assert_equal value, act_value,       msg
+    end
+    assert_equal state, @lex.lex_state,  msg if state
+    assert_equal paren, @lex.paren_nest, msg if paren
+    assert_equal brace, @lex.brace_nest, msg if brace
+  end
+
+  def assert_parse input, exp_sexp
+    assert_equal exp_sexp, processor.class.new.parse(input)
+  end
+
+  def assert_read_escape expected, input
+    @lex.ss.string = input
+    assert_equal expected, @lex.read_escape, input
+  end
+
+  def assert_read_escape_bad input # TODO: rename refute_read_escape
+    @lex.ss.string = input
+    assert_raises RubyParser::SyntaxError do
+      @lex.read_escape
+    end
+  end
+
+  def refute_lexeme
+    x = y = @lex.next_token
+
+    refute x, "not empty: #{y.inspect}"
+  end
+
+  ## Utility Methods:
+
+  def emulate_string_interpolation
+    lex_strterm = lexer.lex_strterm
+    string_nest = lexer.string_nest
+    brace_nest  = lexer.brace_nest
+
+    lexer.string_nest = 0
+    lexer.brace_nest  = 0
+    lexer.cond.push false
+    lexer.cmdarg.push false
+
+    lexer.lex_strterm = nil
+    lexer.lex_state = :expr_beg
+
+    yield
+
+    lexer.lex_state = :expr_endarg
+    assert_next_lexeme :tRCURLY, "}",  :expr_endarg, 0 # 1.9 specific
+
+    lexer.lex_strterm = lex_strterm
+    lexer.lex_state   = :expr_beg
+    lexer.string_nest = string_nest
+    lexer.brace_nest  = brace_nest
+
+    lexer.cond.lexpop
+    lexer.cmdarg.lexpop
+  end
+
+  ## Tests:
+
+  def test_next_token
+    assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
+    assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
+    assert_nil @lex.next_token
+  end
+
+  def test_unicode_ident
+    s = "@\u1088\u1077\u1093\u1072"
+    assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
+  end
+
+  def test_read_escape
+    assert_read_escape "\\",   "\\"
+    assert_read_escape "\n",   "n"
+    assert_read_escape "\t",   "t"
+    assert_read_escape "\r",   "r"
+    assert_read_escape "\f",   "f"
+    assert_read_escape "\13",  "v"
+    assert_read_escape "\0",   "0"
+    assert_read_escape "\07",  "a"
+    assert_read_escape "\007", "a"
+    assert_read_escape "\033", "e"
+    assert_read_escape "\377", "377"
+    assert_read_escape "\377", "xff"
+    assert_read_escape "\010", "b"
+    assert_read_escape " ",    "s"
+    assert_read_escape "q",    "q" # plain vanilla escape
+
+    assert_read_escape "8", "8" # ugh... mri... WHY?!?
+    assert_read_escape "9", "9" # ugh... mri... WHY?!?
+
+    assert_read_escape "$",    "444" # ugh
+  end
+
+  def test_read_escape_c
+    assert_read_escape "\030", "C-x"
+    assert_read_escape "\030", "cx"
+    assert_read_escape "\230", 'C-\M-x'
+    assert_read_escape "\230", 'c\M-x'
+
+    assert_read_escape "\177", "C-?"
+    assert_read_escape "\177", "c?"
+  end
+
+  def test_read_escape_errors
+    assert_read_escape_bad ""
+
+    assert_read_escape_bad "M"
+    assert_read_escape_bad "M-"
+    assert_read_escape_bad "Mx"
+
+    assert_read_escape_bad "Cx"
+    assert_read_escape_bad "C"
+    assert_read_escape_bad "C-"
+
+    assert_read_escape_bad "c"
+  end
+
+  def test_read_escape_m
+    assert_read_escape "\370", "M-x"
+    assert_read_escape "\230", 'M-\C-x'
+    assert_read_escape "\230", 'M-\cx'
+  end
+
+  def test_yylex_ambiguous_uminus
+    assert_lex3("m -3",
+                nil,
+                :tIDENTIFIER, "m", :expr_cmdarg,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tINTEGER,    3,   :expr_end)
+
+    # TODO: verify warning
+  end
+
+  def test_yylex_ambiguous_uplus
+    assert_lex3("m +3",
+                nil,
+                :tIDENTIFIER, "m", :expr_cmdarg,
+                :tINTEGER,    3,   :expr_end)
+
+    # TODO: verify warning
+  end
+
+  def test_yylex_and
+    assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
+  end
+
+  def test_yylex_and2
+    assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
+  end
+
+  def test_yylex_and2_equals
+    assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
+  end
+
+  def test_yylex_and_dot
+    setup_lexer_class RubyParser::V23
+
+    assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
+  end
+
+  def test_yylex_and_dot_call
+    setup_lexer_class RubyParser::V23
+
+    assert_lex3("x&.y", nil,
+                :tIDENTIFIER, "x", :expr_cmdarg,
+                :tLONELY, "&.", :expr_dot,
+                :tIDENTIFIER, "y")
+  end
+
+  def test_yylex_and_dot_call_newline
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("x\n&.y", nil,
+                :tIDENTIFIER, "x", :expr_cmdarg,
+                :tLONELY, "&.", :expr_dot,
+                :tIDENTIFIER, "y")
+  end
+
+  def test_yylex_and_arg
+    self.lex_state = :expr_arg
+
+    assert_lex3(" &y",
+                nil,
+                :tAMPER,      "&", :expr_beg,
+                :tIDENTIFIER, "y", :expr_arg)
+  end
+
+  def test_yylex_and_equals
+    assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
+  end
+
+  def test_yylex_and_expr
+    self.lex_state = :expr_arg
+
+    assert_lex3("x & y",
+                nil,
+                :tIDENTIFIER, "x", :expr_cmdarg,
+                :tAMPER2,     "&", :expr_beg,
+                :tIDENTIFIER, "y", :expr_arg)
+  end
+
+  def test_yylex_and_meth
+    assert_lex_fname "&", :tAMPER2
+  end
+
+  def test_yylex_assoc
+    assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
+  end
+
+  def test_yylex_label__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("{a:",
+                nil,
+                :tLBRACE,     "{", :expr_beg,
+                :tIDENTIFIER, "a", :expr_arg,
+                :tSYMBEG,     ":", :expr_fname)
+  end
+
+  def test_yylex_label_in_params__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("foo(a:",
+                nil,
+                :tIDENTIFIER, "foo", :expr_cmdarg,
+                :tLPAREN2,    "(",   :expr_beg,
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :tSYMBEG,     ":",   :expr_fname)
+  end
+
+  def test_yylex_label__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("{a:",
+                nil,
+                :tLBRACE, "{", :expr_beg,
+                :tLABEL,  "a", :expr_labeled)
+  end
+
+  def test_yylex_label_in_params__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("foo(a:",
+                nil,
+                :tIDENTIFIER, "foo", :expr_cmdarg,
+                :tLPAREN2,    "(",   :expr_beg,
+                :tLABEL,      "a",   :expr_labeled)
+  end
+
+  def test_yylex_paren_string_parens_interpolated
+    setup_lexer('%((#{b}#{d}))',
+                s(:dstr,
+                  "(",
+                  s(:evstr, s(:call, nil, :b)),
+                  s(:evstr, s(:call, nil, :d)),
+                  s(:str, ")")))
+
+    assert_next_lexeme :tSTRING_BEG,     "%)", :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_CONTENT, "(",  :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_DBEG,    nil,  :expr_beg, 0, 0
+
+    emulate_string_interpolation do
+      assert_next_lexeme :tIDENTIFIER,   "b",  :expr_cmdarg, 0, 0
+    end
+
+    assert_next_lexeme :tSTRING_DBEG,    nil,  :expr_beg, 0, 0
+
+    emulate_string_interpolation do
+      assert_next_lexeme :tIDENTIFIER,   "d",  :expr_cmdarg, 0, 0
+    end
+
+    assert_next_lexeme :tSTRING_CONTENT, ")",  :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_END,     ")",  :expr_end, 0, 0
+
+    refute_lexeme
+  end
+
+  def test_yylex_paren_string_interpolated_regexp
+    setup_lexer('%( #{(/abcd/)} )',
+                s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
+
+    assert_next_lexeme :tSTRING_BEG,       "%)",   :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_CONTENT,   " ",    :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_DBEG,      nil,    :expr_beg, 0, 0
+
+    emulate_string_interpolation do
+      assert_next_lexeme :tLPAREN,         "(",    :expr_beg, 1, 0
+      assert_next_lexeme :tREGEXP_BEG,     "/",    :expr_beg, 1, 0
+      assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
+      assert_next_lexeme :tREGEXP_END,     "",     :expr_end, 1, 0
+      assert_next_lexeme :tRPAREN,         ")",    :expr_endfn, 0, 0
+    end
+
+    assert_next_lexeme :tSTRING_CONTENT,   " ",    :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_END,       ")",    :expr_end, 0, 0
+
+    refute_lexeme
+  end
+
+  def test_yylex_not_at_defn__20
+    setup_lexer_class RubyParser::V20
+
+    assert_lex("def +@; end",
+               s(:defn, :+@, s(:args), s(:nil)),
+
+               :kDEF,   "def", :expr_fname, 0, 0,
+               :tUPLUS, "+@",  :expr_arg,   0, 0,
+               :tSEMI,  ";",   :expr_beg,   0, 0,
+               :kEND,   "end", :expr_end,   0, 0)
+
+    assert_lex("def !@; end",
+               s(:defn, :"!@", s(:args), s(:nil)),
+
+               :kDEF,   "def", :expr_fname, 0, 0,
+               :tUBANG, "!@",  :expr_arg,   0, 0,
+               :tSEMI,  ";",   :expr_beg,   0, 0,
+               :kEND,   "end", :expr_end,   0, 0)
+  end
+
+  def test_yylex_not_at_ivar
+    assert_lex("!@ivar",
+               s(:call, s(:ivar, :@ivar), :"!"),
+
+               :tBANG, "!",     :expr_beg, 0, 0,
+               :tIVAR, "@ivar", :expr_end, 0, 0)
+  end
+
+  def test_yylex_number_times_ident_times_return_number
+    assert_lex("1 * b * 3",
+               s(:call,
+                 s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
+                 :*, s(:lit, 3)),
+
+               :tINTEGER,      1, :expr_end, 0, 0,
+               :tSTAR2,      "*", :expr_beg, 0, 0,
+               :tIDENTIFIER, "b", :expr_arg, 0, 0,
+               :tSTAR2,      "*", :expr_beg, 0, 0,
+               :tINTEGER,      3, :expr_end, 0, 0)
+
+    assert_lex("1 * b *\n 3",
+               s(:call,
+                 s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
+                 :*, s(:lit, 3)),
+
+               :tINTEGER,      1, :expr_end, 0, 0,
+               :tSTAR2,      "*", :expr_beg, 0, 0,
+               :tIDENTIFIER, "b", :expr_arg, 0, 0,
+               :tSTAR2,      "*", :expr_beg, 0, 0,
+               :tINTEGER,      3, :expr_end, 0, 0)
+  end
+
+  def test_yylex_paren_string_parens_interpolated_regexp
+    setup_lexer('%((#{(/abcd/)}))',
+                s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
+
+    assert_next_lexeme :tSTRING_BEG,       "%)",   :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_CONTENT,   "(",    :expr_beg, 0, 0
+
+    assert_next_lexeme :tSTRING_DBEG,       nil,   :expr_beg, 0, 0
+
+    emulate_string_interpolation do
+      assert_next_lexeme :tLPAREN,         "(",    :expr_beg, 1, 0
+      assert_next_lexeme :tREGEXP_BEG,     "/",    :expr_beg, 1, 0
+      assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
+      assert_next_lexeme :tREGEXP_END,     "",     :expr_end, 1, 0
+      assert_next_lexeme :tRPAREN,         ")",    :expr_endfn, 0, 0
+    end
+
+    assert_next_lexeme :tSTRING_CONTENT,   ")",    :expr_beg, 0, 0
+    assert_next_lexeme :tSTRING_END,       ")",    :expr_end, 0, 0
+
+    refute_lexeme
+  end
+
+  def test_yylex_method_parens_chevron
+    assert_lex("a()<<1",
+               s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
+               :tIDENTIFIER, "a",   :expr_cmdarg, 0, 0,
+               :tLPAREN2,    "(",   :expr_beg,    1, 0,
+               :tRPAREN,     ")",   :expr_endfn,  0, 0,
+               :tLSHFT,      "<<" , :expr_beg,    0, 0,
+               :tINTEGER,    1,     :expr_end,    0, 0)
+  end
+
+  def test_yylex_lambda_args__20
+    setup_lexer_class RubyParser::V20
+
+    assert_lex("-> (a) { }",
+               s(:iter, s(:call, nil, :lambda),
+                 s(:args, :a)),
+
+               :tLAMBDA,     nil, :expr_endfn,  0, 0,
+               :tLPAREN2,    "(", :expr_beg,    1, 0,
+               :tIDENTIFIER, "a", :expr_arg,    1, 0,
+               :tRPAREN,     ")", :expr_endfn,  0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+  end
+
+  def test_yylex_lambda_as_args_with_block__20
+    setup_lexer_class RubyParser::V20
+
+    assert_lex3("a -> do end do end",
+                nil,
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :tLAMBDA,     nil,   :expr_endfn,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_lambda_args_opt__20
+    setup_lexer_class RubyParser::V20
+
+    assert_lex("-> (a=nil) { }",
+               s(:iter, s(:call, nil, :lambda),
+                 s(:args, s(:lasgn, :a, s(:nil)))),
+
+               :tLAMBDA,     nil, :expr_endfn,  0, 0,
+               :tLPAREN2,    "(", :expr_beg,    1, 0,
+               :tIDENTIFIER, "a", :expr_arg,    1, 0,
+               :tEQL,        "=", :expr_beg,    1, 0,
+               :kNIL,        "nil", :expr_end,    1, 0,
+               :tRPAREN,     ")", :expr_endfn,  0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+  end
+
+  def test_yylex_lambda_hash__20
+    setup_lexer_class RubyParser::V20
+
+    assert_lex("-> (a={}) { }",
+               s(:iter, s(:call, nil, :lambda),
+                 s(:args, s(:lasgn, :a, s(:hash)))),
+
+               :tLAMBDA,     nil, :expr_endfn,  0, 0,
+               :tLPAREN2,    "(", :expr_beg,    1, 0,
+               :tIDENTIFIER, "a", :expr_arg,    1, 0,
+               :tEQL,        "=", :expr_beg,    1, 0,
+               :tLBRACE,     "{", :expr_beg,    1, 1,
+               :tRCURLY,     "}", :expr_endarg, 1, 0,
+               :tRPAREN,     ")", :expr_endfn,  0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+  end
+
+  def test_yylex_iter_array_curly
+    assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
+               s(:iter,
+                 s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
+                 s(:args, :c, :d)),
+
+               :tIDENTIFIER, "f", :expr_cmdarg, 0, 0,
+               :tSYMBOL,     "a", :expr_end,    0, 0,
+               :tCOMMA,      ",", :expr_beg,    0, 0,
+               :tLBRACK,     "[", :expr_beg,    1, 0,
+               :tSYMBOL,     "b", :expr_end,    1, 0,
+               :tRBRACK,     "]", :expr_endarg, 0, 0,
+               :tLBRACE_ARG, "{", :expr_beg,    0, 1,
+               :tPIPE,       "|", :expr_beg,    0, 1,
+               :tIDENTIFIER, "c", :expr_arg,    0, 1,
+               :tCOMMA,      ",", :expr_beg,    0, 1,
+               :tIDENTIFIER, "d", :expr_arg,    0, 1,
+               :tPIPE,       "|", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+  end
+
+  def test_yylex_const_call_same_name
+    assert_lex("X = a { }; b { f :c }",
+               s(:block,
+                 s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
+                 s(:iter,
+                   s(:call, nil, :b),
+                   0,
+                   s(:call, nil, :f, s(:lit, :c)))),
+
+               :tCONSTANT,   "X", :expr_cmdarg, 0, 0,
+               :tEQL,        "=", :expr_beg,    0, 0,
+               :tIDENTIFIER, "a", :expr_arg,    0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0,
+               :tSEMI,       ";", :expr_beg,    0, 0,
+
+               :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tIDENTIFIER, "f", :expr_cmdarg, 0, 1, # different
+               :tSYMBOL,     "c", :expr_end,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+
+    assert_lex("X = a { }; b { X :c }",
+               s(:block,
+                 s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
+                 s(:iter,
+                   s(:call, nil, :b),
+                   0,
+                   s(:call, nil, :X, s(:lit, :c)))),
+
+               :tCONSTANT,   "X", :expr_cmdarg, 0, 0,
+               :tEQL,        "=", :expr_beg,    0, 0,
+               :tIDENTIFIER, "a", :expr_arg,    0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0,
+               :tSEMI,       ";", :expr_beg,    0, 0,
+
+               :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
+               :tLCURLY,     "{", :expr_beg,    0, 1,
+               :tCONSTANT,   "X", :expr_cmdarg, 0, 1, # same
+               :tSYMBOL,     "c", :expr_end,    0, 1,
+               :tRCURLY,     "}", :expr_endarg, 0, 0)
+  end
+
+  def test_yylex_lasgn_call_same_name
+    assert_lex("a = b.c :d => 1",
+               s(:lasgn, :a,
+                 s(:call, s(:call, nil, :b), :c,
+                   s(:hash, s(:lit, :d), s(:lit, 1)))),
+
+               :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
+               :tEQL,        "=", :expr_beg,    0, 0,
+               :tIDENTIFIER, "b", :expr_arg,    0, 0,
+               :tDOT,        ".", :expr_dot,    0, 0,
+               :tIDENTIFIER, "c", :expr_arg,    0, 0, # different
+               :tSYMBOL,     "d", :expr_end,    0, 0,
+               :tASSOC,      "=>", :expr_beg,   0, 0,
+               :tINTEGER,      1, :expr_end,    0, 0)
+
+    assert_lex("a = b.a :d => 1",
+               s(:lasgn, :a,
+                 s(:call, s(:call, nil, :b), :a,
+                   s(:hash, s(:lit, :d), s(:lit, 1)))),
+
+               :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
+               :tEQL,        "=", :expr_beg,    0, 0,
+               :tIDENTIFIER, "b", :expr_arg,    0, 0,
+               :tDOT,        ".", :expr_dot,    0, 0,
+               :tIDENTIFIER, "a", :expr_arg,    0, 0, # same as lvar
+               :tSYMBOL,     "d", :expr_end,    0, 0,
+               :tASSOC,      "=>", :expr_beg,   0, 0,
+               :tINTEGER,      1, :expr_end,    0, 0)
+  end
+
+  def test_yylex_back_ref
+    assert_lex3("[$&, $`, $', $+]",
+                nil,
+                :tLBRACK,   "[",  :expr_beg,
+                :tBACK_REF, :&,   :expr_end, :tCOMMA, ",", :expr_beg,
+                :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
+                :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
+                :tBACK_REF, :+,   :expr_end,
+                :tRBRACK,   "]",  :expr_endarg)
+  end
+
+  def test_yylex_backslash
+    assert_lex3("1 \\\n+ 2",
+                nil,
+                :tINTEGER, 1,   :expr_end,
+                :tPLUS,    "+", :expr_beg,
+                :tINTEGER, 2,   :expr_end)
+  end
+
+  def test_yylex_backslash_bad
+    refute_lex("1 \\ + 2", :tINTEGER, 1)
+  end
+
+  def test_yylex_backtick
+    assert_lex3("`ls`",
+                nil,
+                :tXSTRING_BEG,    "`",  :expr_beg,
+                :tSTRING_CONTENT, "ls", :expr_beg,
+                :tSTRING_END,     "`",  :expr_end)
+  end
+
+  def test_yylex_backtick_cmdarg
+    self.lex_state = :expr_dot
+
+    # \n ensures expr_cmd (TODO: why?)
+    assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
+  end
+
+  def test_yylex_backtick_dot
+    self.lex_state = :expr_dot
+
+    assert_lex3("a.`(3)",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tDOT,        ".", :expr_dot,
+                :tBACK_REF2,  "`", :expr_arg,
+                :tLPAREN2,    "(", :expr_beg,
+                :tINTEGER,    3,   :expr_end,
+                :tRPAREN,     ")", :expr_endfn)
+  end
+
+  def test_yylex_backtick_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("`",
+                nil,
+                :tBACK_REF2, "`", :expr_end)
+  end
+
+  def test_yylex_bad_char
+    refute_lex(" \010 ")
+  end
+
+  def test_yylex_bang
+    assert_lex3("!", nil, :tBANG, "!", :expr_beg)
+  end
+
+  def test_yylex_bang_equals
+    assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
+  end
+
+  def test_yylex_bang_tilde
+    assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
+  end
+
+  def test_yylex_block_bug_1
+    assert_lex3("a do end",
+                s(:iter, s(:call, nil, :a), 0),
+
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_block_bug_2
+    assert_lex3("a = 1\na do\nend",
+                s(:block,
+                  s(:lasgn, :a, s(:lit, 1)),
+                  s(:iter, s(:call, nil, :a), 0)),
+
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :tEQL,        "=",   :expr_beg,
+                :tINTEGER,    1,     :expr_end,
+                :tNL,         nil,   :expr_beg,
+                :tIDENTIFIER, "a",   :expr_cmdarg,
+                :kDO,         "do",  :expr_beg,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_block_bug_3
+    assert_lex3("a { }",
+                s(:iter, s(:call, nil, :a), 0),
+
+                :tIDENTIFIER, "a", :expr_cmdarg, # verified
+                :tLCURLY,     "{", :expr_beg, # TODO: expr_beg|expr_label
+                :tRCURLY,     "}", :expr_endarg)
+  end
+
+  def test_yylex_carat
+    assert_lex3("^", nil, :tCARET, "^", :expr_beg)
+  end
+
+  def test_yylex_carat_equals
+    assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
+  end
+
+  def test_yylex_colon2
+    assert_lex3("A::B",
+                nil,
+                :tCONSTANT, "A",  :expr_cmdarg,
+                :tCOLON2,   "::", :expr_dot,
+                :tCONSTANT, "B",  :expr_arg)
+  end
+
+  def test_yylex_colon2_argh
+    assert_lex3("module X::Y\n  c\nend",
+                nil,
+                :kMODULE,     "module", :expr_value,
+                :tCONSTANT,   "X",      :expr_arg,
+                :tCOLON2,     "::",     :expr_dot,
+                :tCONSTANT,   "Y",      :expr_arg,
+                :tNL,         nil,      :expr_beg,
+                :tIDENTIFIER, "c",      :expr_cmdarg,
+                :tNL,         nil,      :expr_beg,
+                :kEND,        "end",    :expr_end)
+  end
+
+  def test_yylex_colon3
+    assert_lex3("::Array",
+                nil,
+                :tCOLON3,   "::",    :expr_beg,
+                :tCONSTANT, "Array", :expr_arg)
+  end
+
+  def test_yylex_comma
+    assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
+  end
+
+  def test_yylex_comment
+    assert_lex3("1 # one\n# two\n2",
+                nil,
+                :tINTEGER, 1,   :expr_end,
+                :tNL,      nil, :expr_beg,
+                :tINTEGER, 2,   :expr_end)
+
+    assert_equal "# one\n# two\n", @lex.comments
+  end
+
+  def test_yylex_comment_begin
+    assert_lex3("=begin\nblah\nblah\n=end\n42",
+                nil,
+                :tINTEGER, 42, :expr_end)
+
+    assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
+  end
+
+  def test_yylex_comment_begin_bad
+    refute_lex("=begin\nblah\nblah\n")
+
+    assert_equal "", @lex.comments
+  end
+
+  def test_yylex_comment_begin_not_comment
+    assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
+                nil,
+                :tIDENTIFIER, "beginfoo", :expr_cmdarg,
+                :tEQL,        "=",        :expr_beg,
+                :tINTEGER,    5,          :expr_end,
+                :tNL,         nil,        :expr_beg,
+                :tIDENTIFIER, "p",        :expr_cmdarg,
+                :tIDENTIFIER, "x",        :expr_arg,
+                :tEQL,        "=",        :expr_beg,
+                :tIDENTIFIER, "beginfoo", :expr_arg)
+  end
+
+  def test_yylex_comment_begin_space
+    assert_lex3("=begin blah\nblah\n=end\n", nil)
+
+    assert_equal "=begin blah\nblah\n=end\n", @lex.comments
+  end
+
+  def test_yylex_comment_end_space_and_text
+    assert_lex3("=begin blah\nblah\n=end blab\n", nil)
+
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+  end
+
+  def test_yylex_comment_eos
+    assert_lex3("# comment", nil)
+  end
+
+  def test_yylex_constant
+    assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
+  end
+
+  def test_yylex_constant_semi
+    assert_lex3("ArgumentError;",
+                nil,
+                :tCONSTANT, "ArgumentError", :expr_cmdarg,
+                :tSEMI,     ";",             :expr_beg)
+  end
+
+  def test_yylex_cvar
+    assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
+  end
+
+  def test_yylex_cvar_bad
+    assert_raises RubyParser::SyntaxError do
+      assert_lex3("@@1", nil)
+    end
+  end
+
+  def test_yylex_def_bad_name
+    self.lex_state = :expr_fname
+    refute_lex("def [ ", :kDEF, "def")
+  end
+
+  def test_yylex_div
+    assert_lex3("a / 2",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tDIVIDE,     "/", :expr_beg,
+                :tINTEGER,    2,   :expr_end)
+  end
+
+  def test_yylex_div_equals
+    assert_lex3("a /= 2",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tOP_ASGN,    "/", :expr_beg,
+                :tINTEGER,    2,   :expr_end)
+  end
+
+  def test_yylex_do
+    assert_lex3("x do 42 end",
+                nil,
+                :tIDENTIFIER, "x",   :expr_cmdarg,
+                :kDO,         "do",  :expr_beg,
+                :tINTEGER,    42,    :expr_end,
+                :kEND,        "end", :expr_end)
+  end
+
+  def test_yylex_do_block
+    self.lex_state = :expr_endarg
+
+    assert_lex3("x.y do 42 end",
+                nil,
+                :tIDENTIFIER, "x",   :expr_end,
+                :tDOT,        ".",   :expr_dot,
+                :tIDENTIFIER, "y",   :expr_arg,
+                :kDO_BLOCK,   "do",  :expr_beg,
+                :tINTEGER,    42,    :expr_end,
+                :kEND,        "end", :expr_end) do
+      @lex.cmdarg.push true
+    end
+  end
+
+  def test_yylex_do_block2
+    self.lex_state = :expr_endarg
+
+    assert_lex3("do 42 end",
+                nil,
+                :kDO_BLOCK, "do",  :expr_beg,
+                :tINTEGER,  42,    :expr_end,
+                :kEND,      "end", :expr_end)
+  end
+
+  def test_yylex_is_your_spacebar_broken?
+    assert_lex3(":a!=:b",
+                nil,
+                :tSYMBOL, "a",  :expr_end,
+                :tNEQ,    "!=", :expr_beg,
+                :tSYMBOL, "b",  :expr_end)
+  end
+
+  def test_yylex_do_cond
+    assert_lex3("x do 42 end",
+                nil,
+                :tIDENTIFIER, "x",   :expr_cmdarg,
+                :kDO_COND,    "do",  :expr_beg,
+                :tINTEGER,    42,    :expr_end,
+                :kEND,        "end", :expr_end) do
+      @lex.cond.push true
+    end
+  end
+
+  def test_yylex_dollar_bad
+    e = refute_lex("$%")
+    assert_includes(e.message, "is not allowed as a global variable name")
+  end
+
+  def test_yylex_dollar_eos
+    assert_lex3("$", nil, "$", "$", :expr_end) # FIX: wtf is this?!?
+  end
+
+  def test_yylex_dot # HINT message sends
+    assert_lex3(".", nil, :tDOT, ".", :expr_dot)
+  end
+
+  def test_yylex_dot2
+    assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
+  end
+
+  def test_yylex_dot3
+    assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
+  end
+
+  def test_yylex_equals
+    # FIX: this sucks
+    assert_lex3("=", nil, :tEQL, "=", :expr_beg)
+  end
+
+  def test_yylex_equals2
+    assert_lex3("==", nil, :tEQ, "==", :expr_beg)
+  end
+
+  def test_yylex_equals3
+    assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
+  end
+
+  def test_yylex_equals_tilde
+    assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
+  end
+
+  def test_yylex_float
+    assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
+  end
+
+  def test_yylex_float_bad_no_underscores
+    refute_lex "1__0.0"
+  end
+
+  def test_yylex_float_bad_no_zero_leading
+    refute_lex ".0"
+  end
+
+  def test_yylex_float_bad_trailing_underscore
+    refute_lex "123_.0"
+  end
+
+  def test_yylex_float_call
+    assert_lex3("1.0.to_s",
+                nil,
+                :tFLOAT,      1.0,    :expr_end,
+                :tDOT,        ".",    :expr_dot,
+                :tIDENTIFIER, "to_s", :expr_arg)
+  end
+
+  def test_yylex_float_dot_E
+    assert_lex3("1.0E10",
+                nil,
+                :tFLOAT, 10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_dot_E_neg
+    assert_lex3("-1.0E10",
+                nil,
+                :tUMINUS_NUM, "-",           :expr_beg,
+                :tFLOAT,      10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_dot_e
+    assert_lex3("1.0e10",
+                nil,
+                :tFLOAT, 10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_dot_e_neg
+    assert_lex3("-1.0e10",
+                nil,
+                :tUMINUS_NUM, "-",           :expr_beg,
+                :tFLOAT,      10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_e
+    assert_lex3("1e10",
+                nil,
+                :tFLOAT, 10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_e_bad_double_e
+    assert_lex3("1e2e3",
+                nil,
+                :tFLOAT, 100, :expr_end,
+                :tIDENTIFIER, "e3", :expr_end)
+  end
+
+  def test_yylex_float_if_modifier
+    assert_lex3("1e2if",
+                nil,
+                :tFLOAT, 100, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+  end
+
+  def test_yylex_float_e_bad_trailing_underscore
+    refute_lex "123_e10"
+  end
+
+  def test_yylex_float_e_minus
+    assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
+  end
+
+  def test_yylex_float_e_neg
+    assert_lex3("-1e10",
+                nil,
+                :tUMINUS_NUM, "-",           :expr_beg,
+                :tFLOAT,      10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_e_neg_minus
+    assert_lex3("-1e-10",
+                nil,
+                :tUMINUS_NUM, "-",     :expr_beg,
+                :tFLOAT,      1.0e-10, :expr_end)
+  end
+
+  def test_yylex_float_e_neg_plus
+    assert_lex3("-1e+10",
+                nil,
+                :tUMINUS_NUM, "-",           :expr_beg,
+                :tFLOAT,      10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_e_plus
+    assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
+  end
+
+  def test_yylex_float_e_zero
+    assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
+  end
+
+  def test_yylex_float_neg
+    assert_lex3("-1.0",
+                nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tFLOAT,      1.0, :expr_end)
+  end
+
+  def test_yylex_ge
+    assert_lex3("a >= 2",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tGEQ,        ">=", :expr_beg,
+                :tINTEGER,    2,    :expr_end)
+  end
+
+  def test_yylex_global
+    assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
+  end
+
+  def test_yylex_global_backref
+    self.lex_state = :expr_fname
+
+    assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
+  end
+
+  def test_yylex_global_dash_nothing
+    assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
+  end
+
+  def test_yylex_global_dash_something
+    assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
+  end
+
+  def test_yylex_global_number
+    self.lex_state = :expr_fname
+
+    assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
+  end
+
+  def test_yylex_global_number_big
+    self.lex_state = :expr_fname
+
+    assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
+  end
+
+  def test_yylex_global_other
+    assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
+                nil,
+                :tLBRACK, "[",   :expr_beg,
+                :tGVAR,   "$~",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$*",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$$",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$?",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$!",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$@",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$/",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$\\", :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$;",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$,",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$.",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$=",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$:",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$<",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$>",  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tGVAR,   "$\"", :expr_end,
+                :tRBRACK, "]",   :expr_endarg)
+  end
+
+  def test_yylex_global_underscore
+    assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
+  end
+
+  def test_yylex_global_wierd
+    assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
+  end
+
+  def test_yylex_global_zero
+    assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
+  end
+
+  def test_yylex_gt
+    assert_lex3("a > 2",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tGT,         ">", :expr_beg,
+                :tINTEGER,    2,   :expr_end)
+  end
+
+  def test_yylex_heredoc_backtick
+    assert_lex3("a = <<`EOF`\n  blah blah\nEOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",             :expr_cmdarg,
+                :tEQL,            "=",             :expr_beg,
+                :tXSTRING_BEG,    "`",             :expr_beg,
+                :tSTRING_CONTENT, "  blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",           :expr_end,
+                :tNL,             nil,             :expr_beg)
+  end
+
+  def test_yylex_heredoc_double
+    assert_lex3("a = <<\"EOF\"\n  blah blah\nEOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",             :expr_cmdarg,
+                :tEQL,            "=",             :expr_beg,
+                :tSTRING_BEG,     "\"",            :expr_beg,
+                :tSTRING_CONTENT, "  blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",           :expr_end,
+                :tNL,             nil,             :expr_beg)
+  end
+
+  def test_yylex_heredoc_double_dash
+    assert_lex3("a = <<-\"EOF\"\n  blah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",             :expr_cmdarg,
+                :tEQL,            "=",             :expr_beg,
+                :tSTRING_BEG,     "\"",            :expr_beg,
+                :tSTRING_CONTENT, "  blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",           :expr_end,
+                :tNL,             nil,             :expr_beg)
+  end
+
+  def test_yylex_heredoc_double_squiggly
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("a = <<~\"EOF\"\n  blah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",           :expr_cmdarg,
+                :tEQL,            "=",           :expr_beg,
+                :tSTRING_BEG,     "\"",          :expr_beg,
+                :tSTRING_CONTENT, "blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",         :expr_end,
+                :tNL,             nil,           :expr_beg)
+  end
+
+  # mri handles tabs in a pretty specific way:
+  # https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
+  def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("a = <<~\"EOF\"\n  blah blah\n \tblah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",                        :expr_cmdarg,
+                :tEQL,            "=",                        :expr_beg,
+                :tSTRING_BEG,     "\"",                       :expr_beg,
+                :tSTRING_CONTENT, "blah blah\n\tblah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",                      :expr_end,
+                :tNL,             nil,                        :expr_beg)
+  end
+
+  def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("a = <<~\"EOF\"\n        blah blah\n\t blah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",                       :expr_cmdarg,
+                :tEQL,            "=",                       :expr_beg,
+                :tSTRING_BEG,     "\"",                      :expr_beg,
+                :tSTRING_CONTENT, "blah blah\n blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",                     :expr_end,
+                :tNL,             nil,                       :expr_beg)
+  end
+
+  def test_yylex_heredoc_double_eos
+    refute_lex("a = <<\"EOF\"\nblah",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_double_eos_nl
+    refute_lex("a = <<\"EOF\"\nblah\n",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_double_interp
+    assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",     :expr_cmdarg,
+                :tEQL,            "=",     :expr_beg,
+                :tSTRING_BEG,     "\"",    :expr_beg,
+                :tSTRING_CONTENT, "#x a ", :expr_beg,
+                :tSTRING_DVAR,    "\#@",   :expr_beg,
+                :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
+                :tSTRING_DVAR,    "\#$",   :expr_beg,
+                :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
+                :tSTRING_DBEG,    "\#{",   :expr_beg,
+                :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
+                :tSTRING_END,     "EOF",   :expr_end,
+                :tNL,             nil,     :expr_beg)
+  end
+
+  def test_yylex_heredoc_empty
+    assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
+                nil,
+                :tSTRING_BEG,     "\"",          :expr_beg,
+                :tSTRING_DBEG,    "\#{",         :expr_beg,
+                :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
+                :tSTRING_END,     "",            :expr_end,
+                :tNL,             nil,           :expr_beg)
+  end
+
+  def test_yylex_heredoc_none
+    assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
+                nil,
+                :tIDENTIFIER,     "a",            :expr_cmdarg,
+                :tEQL,            "=",            :expr_beg,
+                :tSTRING_BEG,     "\"",           :expr_beg,
+                :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
+                :tSTRING_END,     "EOF",          :expr_end,
+                :tNL,             nil,            :expr_beg)
+  end
+
+  def test_yylex_heredoc_none_bad_eos
+    refute_lex("a = <<EOF",
+                   :tIDENTIFIER, "a",
+                   :tEQL,        "=",
+                   :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_none_dash
+    assert_lex3("a = <<-EOF\nblah\nblah\n  EOF\n",
+                nil,
+                :tIDENTIFIER,     "a",            :expr_cmdarg,
+                :tEQL,            "=",            :expr_beg,
+                :tSTRING_BEG,     "\"",           :expr_beg,
+                :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
+                :tSTRING_END,     "EOF",          :expr_end,
+                :tNL,             nil,            :expr_beg)
+  end
+
+  def test_yylex_heredoc_none_squiggly
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("a = <<~EOF\n  blah\n  blah\n  EOF\n",
+                nil,
+                :tIDENTIFIER,     "a",            :expr_cmdarg,
+                :tEQL,            "=",            :expr_beg,
+                :tSTRING_BEG,     "\"",           :expr_beg,
+                :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
+                :tSTRING_END,     "EOF",          :expr_end,
+                :tNL,             nil,            :expr_beg)
+  end
+
+  def test_yylex_heredoc_single
+    assert_lex3("a = <<'EOF'\n  blah blah\nEOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",             :expr_cmdarg,
+                :tEQL,            "=",             :expr_beg,
+                :tSTRING_BEG,     "\"",            :expr_beg,
+                :tSTRING_CONTENT, "  blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",           :expr_end,
+                :tNL,             nil,             :expr_beg)
+  end
+
+  def test_yylex_heredoc_single_bad_eos_body
+    refute_lex("a = <<'EOF'\nblah",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_empty
+    refute_lex("a = <<''\n",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_term
+    refute_lex("a = <<'EOF",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_single_bad_eos_term_nl
+    refute_lex("a = <<'EOF\ns = 'blah blah'",
+               :tIDENTIFIER, "a",
+               :tEQL,        "=",
+               :tSTRING_BEG, "\"")
+  end
+
+  def test_yylex_heredoc_single_dash
+    assert_lex3("a = <<-'EOF'\n  blah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",             :expr_cmdarg,
+                :tEQL,            "=",             :expr_beg,
+                :tSTRING_BEG,     "\"",            :expr_beg,
+                :tSTRING_CONTENT, "  blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",           :expr_end,
+                :tNL,             nil,             :expr_beg)
+  end
+
+  def test_yylex_heredoc_single_squiggly
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("a = <<~'EOF'\n  blah blah\n  EOF\n\n",
+                nil,
+                :tIDENTIFIER,     "a",           :expr_cmdarg,
+                :tEQL,            "=",           :expr_beg,
+                :tSTRING_BEG,     "\"",          :expr_beg,
+                :tSTRING_CONTENT, "blah blah\n", :expr_beg,
+                :tSTRING_END,     "EOF",         :expr_end,
+                :tNL,             nil,           :expr_beg)
+  end
+
+  def test_yylex_identifier
+    assert_lex3("identifier",
+                nil,
+                :tIDENTIFIER, "identifier", :expr_cmdarg)
+  end
+
+  def test_yylex_identifier_bang
+    assert_lex3("identifier!",
+                nil,
+                :tFID, "identifier!", :expr_cmdarg)
+  end
+
+  def test_yylex_identifier_cmp
+    assert_lex_fname "<=>", :tCMP
+  end
+
+  def test_yylex_identifier_def__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
+  end
+
+  def test_yylex_identifier_def__1920
+    setup_lexer_class RubyParser::V19
+
+    assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
+  end
+
+  def test_yylex_identifier_eh
+    assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
+  end
+
+  def test_yylex_identifier_equals_arrow
+    assert_lex3(":blah==>",
+                nil,
+                :tSYMBOL, "blah=", :expr_end,
+                :tASSOC,  "=>",    :expr_beg)
+  end
+
+  def test_yylex_identifier_equals3
+    assert_lex3(":a===b",
+                nil,
+                :tSYMBOL,     "a",   :expr_end,
+                :tEQQ,        "===", :expr_beg,
+                :tIDENTIFIER, "b",   :expr_arg)
+  end
+
+  def test_yylex_identifier_equals_equals_arrow
+    assert_lex3(":a==>b",
+                nil,
+                :tSYMBOL, "a=", :expr_end,
+                :tASSOC, "=>", :expr_beg,
+                :tIDENTIFIER, "b", :expr_arg)
+  end
+
+  def test_yylex_identifier_equals_caret
+    assert_lex_fname "^", :tCARET
+  end
+
+  def test_yylex_identifier_equals_def__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
+  end
+
+  def test_yylex_identifier_equals_def__1920
+    setup_lexer_class RubyParser::V19
+
+    assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
+  end
+
+  def test_yylex_identifier_equals_def2
+    assert_lex_fname "==", :tEQ
+  end
+
+  def test_yylex_identifier_equals_expr
+    self.lex_state = :expr_dot
+    assert_lex3("y = arg",
+                nil,
+                :tIDENTIFIER, "y",   :expr_cmdarg,
+                :tEQL,        "=",   :expr_beg,
+                :tIDENTIFIER, "arg", :expr_arg)
+  end
+
+  def test_yylex_identifier_equals_or
+    assert_lex_fname "|", :tPIPE
+  end
+
+  def test_yylex_identifier_equals_slash
+    assert_lex_fname "/", :tDIVIDE
+  end
+
+  def test_yylex_identifier_equals_tilde
+    self.lex_state = :expr_fname # can only set via parser's defs
+
+    assert_lex3("identifier=~",
+                nil,
+                :tIDENTIFIER, "identifier", :expr_endfn,
+                :tMATCH,      "=~",         :expr_beg)
+  end
+
+  def test_yylex_identifier_gt
+    assert_lex_fname ">", :tGT
+  end
+
+  def test_yylex_identifier_le
+    assert_lex_fname "<=", :tLEQ
+  end
+
+  def test_yylex_identifier_lt
+    assert_lex_fname "<", :tLT
+  end
+
+  def test_yylex_identifier_tilde
+    assert_lex_fname "~", :tTILDE
+  end
+
+  def test_yylex_index
+    assert_lex_fname "[]", :tAREF
+  end
+
+  def test_yylex_index_equals
+    assert_lex_fname "[]=", :tASET
+  end
+
+  def test_yylex_integer
+    assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_bin
+    assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_bin_bad_none
+    refute_lex "0b "
+  end
+
+  def test_yylex_integer_bin_bad_underscores
+    refute_lex "0b10__01"
+  end
+
+  def test_yylex_integer_dec
+    assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_dec_bad_underscores
+    refute_lex "42__24"
+  end
+
+  def test_yylex_integer_dec_d
+    assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_dec_d_bad_none
+    refute_lex "0d"
+  end
+
+  def test_yylex_integer_dec_d_bad_underscores
+    refute_lex "0d42__24"
+  end
+
+  def test_yylex_integer_if_modifier
+    assert_lex3("123if",
+                nil,
+                :tINTEGER, 123, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+  end
+
+  def test_yylex_question_eh_a__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
+  end
+
+  def test_yylex_question_eh_a__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
+  end
+
+  def test_yylex_question_eh_escape_M_escape_C__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
+  end
+
+  def test_yylex_question_eh_escape_M_escape_C__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
+  end
+
+  def test_yylex_integer_hex
+    assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
+  end
+
+  def test_yylex_integer_hex_bad_none
+    refute_lex "0x "
+  end
+
+  def test_yylex_integer_hex_bad_underscores
+    refute_lex "0xab__cd"
+  end
+
+  def test_yylex_integer_oct
+    assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_oct_bad_range
+    refute_lex "08"
+  end
+
+  def test_yylex_integer_oct_bad_range2
+    refute_lex "08"
+  end
+
+  def test_yylex_integer_oct_bad_underscores
+    refute_lex "01__23"
+  end
+
+  def test_yylex_integer_oct_O
+    assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
+  end
+
+  def test_yylex_integer_oct_O_bad_range
+    refute_lex "0O8"
+  end
+
+  def test_yylex_integer_oct_O_bad_underscores
+    refute_lex "0O1__23"
+  end
+
+  def test_yylex_integer_oct_O_not_bad_none
+    assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
+  end
+
+  def test_yylex_integer_oct_o
+    assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
+  end
+
+  def test_yylex_integer_oct_o_bad_range
+    refute_lex "0o8"
+  end
+
+  def test_yylex_integer_oct_o_bad_underscores
+    refute_lex "0o1__23"
+  end
+
+  def test_yylex_integer_oct_o_not_bad_none
+    assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
+  end
+
+  def test_yylex_integer_trailing
+    assert_lex3("1.to_s",
+                nil,
+                :tINTEGER,    1,      :expr_end,
+                :tDOT,        ".",    :expr_dot,
+                :tIDENTIFIER, "to_s", :expr_arg)
+  end
+
+  def test_yylex_integer_underscore
+    assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_integer_underscore_bad
+    refute_lex "4__2"
+  end
+
+  def test_yylex_integer_zero
+    assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
+  end
+
+  def test_yylex_ivar
+    assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
+  end
+
+  def test_yylex_ivar_bad
+    refute_lex "@1"
+  end
+
+  def test_yylex_ivar_bad_0_length
+    refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
+  end
+
+  def test_yylex_keyword_expr
+    self.lex_state = :expr_endarg
+
+    assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
+  end
+
+  def test_yylex_lt
+    assert_lex3("<", nil, :tLT, "<", :expr_beg)
+  end
+
+  def test_yylex_lt2
+    assert_lex3("a << b",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tLSHFT,      "<<", :expr_beg,
+                :tIDENTIFIER, "b",  :expr_arg)
+  end
+
+  def test_yylex_lt2_equals
+    assert_lex3("a <<= b",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tOP_ASGN,    "<<", :expr_beg,
+                :tIDENTIFIER, "b",  :expr_arg)
+  end
+
+  def test_yylex_lt_equals
+    assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
+  end
+
+  def test_yylex_minus
+    assert_lex3("1 - 2",
+                nil,
+                :tINTEGER, 1,   :expr_end,
+                :tMINUS,   "-", :expr_beg,
+                :tINTEGER, 2,   :expr_end)
+  end
+
+  def test_yylex_minus_equals
+    assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
+  end
+
+  def test_yylex_minus_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
+  end
+
+  def test_yylex_minus_unary_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
+  end
+
+  def test_yylex_minus_unary_number
+    assert_lex3("-42",
+                nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tINTEGER,    42,  :expr_end)
+  end
+
+  def test_yylex_nth_ref
+    assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
+               nil,
+               :tLBRACK,  "[", :expr_beg,
+               :tNTH_REF, 1,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 2,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 3,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 4,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 5,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 6,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 7,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 8,   :expr_end, :tCOMMA,   ",", :expr_beg,
+               :tNTH_REF, 9,   :expr_end,
+               :tRBRACK,  "]", :expr_endarg)
+  end
+
+  def test_yylex_open_bracket
+    assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
+  end
+
+  def test_yylex_open_bracket_cmdarg
+    self.lex_state = :expr_cmdarg
+
+    assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
+  end
+
+  def test_yylex_open_bracket_exprarg__18
+    setup_lexer_class RubyParser::V18
+    self.lex_state = :expr_arg
+
+    assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
+  end
+
+  def test_yylex_open_bracket_exprarg__19
+    setup_lexer_class RubyParser::V19
+    self.lex_state = :expr_arg
+
+    assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
+  end
+
+  def test_yylex_open_curly_bracket
+    assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
+  end
+
+  def test_yylex_open_curly_bracket_arg
+    self.lex_state = :expr_arg
+
+    assert_lex3("m { 3 }",
+                nil,
+                :tIDENTIFIER, "m", :expr_cmdarg,
+                :tLCURLY,     "{", :expr_beg,
+                :tINTEGER,    3,   :expr_end,
+                :tRCURLY,     "}", :expr_endarg)
+  end
+
+  def test_yylex_open_curly_bracket_block
+    self.lex_state = :expr_endarg # seen m(3)
+
+    assert_lex3("{ 4 }",
+                nil,
+                :tLBRACE_ARG, "{", :expr_beg,
+                :tINTEGER,    4,   :expr_end,
+                :tRCURLY,     "}", :expr_endarg)
+  end
+
+  def test_yylex_open_square_bracket_arg
+    self.lex_state = :expr_arg
+
+    assert_lex3("m [ 3 ]",
+                nil,
+                :tIDENTIFIER, "m", :expr_cmdarg,
+                :tLBRACK,     "[", :expr_beg,
+                :tINTEGER,    3,   :expr_end,
+                :tRBRACK,     "]", :expr_endarg)
+  end
+
+  def test_yylex_open_square_bracket_ary
+    assert_lex3("[1, 2, 3]",
+                nil,
+                :tLBRACK, "[", :expr_beg,
+                :tINTEGER, 1,  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tINTEGER, 2,  :expr_end, :tCOMMA,  ",", :expr_beg,
+                :tINTEGER, 3,  :expr_end,
+                :tRBRACK, "]", :expr_endarg)
+  end
+
+  def test_yylex_open_square_bracket_meth
+    assert_lex3("m[3]",
+               nil,
+               :tIDENTIFIER, "m", :expr_cmdarg,
+               :tLBRACK2,    "[", :expr_beg,
+               :tINTEGER,    3,   :expr_end,
+               :tRBRACK,     "]", :expr_endarg)
+  end
+
+  def test_yylex_or
+    assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
+  end
+
+  def test_yylex_or2
+    assert_lex3("||", nil, :tOROP, "||", :expr_beg)
+  end
+
+  def test_yylex_or2_equals
+    assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
+  end
+
+  def test_yylex_or_equals
+    assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
+  end
+
+  def test_yylex_percent
+    assert_lex3("a % 2",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tPERCENT,    "%", :expr_beg,
+                :tINTEGER,    2,   :expr_end)
+  end
+
+  def test_yylex_percent_equals
+    assert_lex3("a %= 2",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tOP_ASGN,    "%", :expr_beg,
+                :tINTEGER,    2,   :expr_end)
+  end
+
+  def test_yylex_plus
+    assert_lex3("1 + 1", # TODO lex_state?
+                nil,
+                :tINTEGER, 1,   :expr_end,
+                :tPLUS,    "+", :expr_beg,
+                :tINTEGER, 1,   :expr_end)
+  end
+
+  def test_yylex_plus_equals
+    assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
+  end
+
+  def test_yylex_plus_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
+  end
+
+  def test_yylex_plus_unary_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
+  end
+
+  def test_yylex_not_unary_method
+    self.lex_state = :expr_fname
+
+    assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
+  end
+
+  def test_yylex_numbers
+    assert_lex3("0b10", nil, :tINTEGER, 2,  :expr_end)
+    assert_lex3("0B10", nil, :tINTEGER, 2,  :expr_end)
+
+    assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
+    assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
+
+    assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
+    assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
+
+    assert_lex3("0o10", nil, :tINTEGER, 8,  :expr_end)
+    assert_lex3("0O10", nil, :tINTEGER, 8,  :expr_end)
+
+    assert_lex3("0o",   nil, :tINTEGER, 0,  :expr_end)
+    assert_lex3("0O",   nil, :tINTEGER, 0,  :expr_end)
+
+    assert_lex3("0",    nil, :tINTEGER, 0,  :expr_end)
+
+    refute_lex "0x"
+    refute_lex "0X"
+    refute_lex "0b"
+    refute_lex "0B"
+    refute_lex "0d"
+    refute_lex "0D"
+
+    refute_lex "08"
+    refute_lex "09"
+    refute_lex "0o8"
+    refute_lex "0o9"
+    refute_lex "0O8"
+    refute_lex "0O9"
+
+    refute_lex "1_e1"
+    refute_lex "1_.1"
+    refute_lex "1__1"
+  end
+
+  def test_yylex_plus_unary_number
+    assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_question__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
+  end
+
+  def test_yylex_question__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
+  end
+
+  def test_yylex_question_bad_eos
+    refute_lex "?"
+  end
+
+  def test_yylex_question_ws
+    assert_lex3("? ",  nil, :tEH, "?", :expr_value)
+    assert_lex3("?\n", nil, :tEH, "?", :expr_value)
+    assert_lex3("?\t", nil, :tEH, "?", :expr_value)
+    assert_lex3("?\v", nil, :tEH, "?", :expr_value)
+    assert_lex3("?\r", nil, :tEH, "?", :expr_value)
+    assert_lex3("?\f", nil, :tEH, "?", :expr_value)
+  end
+
+  def test_yylex_question_ws_backslashed__18
+    setup_lexer_class RubyParser::V18
+
+    assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
+    assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
+    assert_lex3("?\\t", nil, :tINTEGER,  9, :expr_end)
+    assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
+    assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
+    assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
+  end
+
+  def test_yylex_question_ws_backslashed__19
+    setup_lexer_class RubyParser::V19
+
+    assert_lex3("?\\ ", nil, :tSTRING, " ",  :expr_end)
+    assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
+    assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
+    assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
+    assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
+    assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
+  end
+
+  def test_yylex_rbracket
+    assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
+  end
+
+  def test_yylex_rcurly
+    assert_lex("}", nil, :tRCURLY, "}", :expr_endarg, 0, 1) do
+      lexer.brace_nest += 2
+    end
+  end
+
+  def test_yylex_regexp
+    assert_lex3("/regexp/",
+                nil,
+                :tREGEXP_BEG,     "/",      :expr_beg,
+                :tSTRING_CONTENT, "regexp", :expr_beg,
+                :tREGEXP_END,     "",       :expr_end)
+  end
+
+  def test_yylex_regexp_ambiguous
+    assert_lex3("method /regexp/",
+                nil,
+                :tIDENTIFIER,     "method", :expr_cmdarg,
+                :tREGEXP_BEG,     "/",      :expr_cmdarg,
+                :tSTRING_CONTENT, "regexp", :expr_cmdarg,
+                :tREGEXP_END,     "",       :expr_end)
+  end
+
+  def test_yylex_regexp_bad
+    refute_lex("/.*/xyz",
+               :tREGEXP_BEG,     "/",
+               :tSTRING_CONTENT, ".*")
+  end
+
+  def test_yylex_regexp_escape_C
+    assert_lex3("/regex\\C-x/",
+                nil,
+                :tREGEXP_BEG,     "/",          :expr_beg,
+                :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
+                :tREGEXP_END,     "",           :expr_end)
+  end
+
+  def test_yylex_regexp_escape_C_M
+    assert_lex3("/regex\\C-\\M-x/",
+                nil,
+                :tREGEXP_BEG,     "/",              :expr_beg,
+                :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
+                :tREGEXP_END,     "",               :expr_end)
+  end
+
+  def test_yylex_regexp_escape_C_M_craaaazy
+    assert_lex3("/regex\\C-\\\n\\M-x/",
+                nil,
+                :tREGEXP_BEG,     "/",              :expr_beg,
+                :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
+                :tREGEXP_END,     "",               :expr_end)
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash
+    refute_lex '/regex\\Cx/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash_eos
+    refute_lex '/regex\\C-/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_dash_eos2
+    refute_lex '/regex\\C-', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_eos
+    refute_lex '/regex\\C/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_C_bad_eos2
+    refute_lex '/regex\\c', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M
+    assert_lex3("/regex\\M-x/",
+                nil,
+                :tREGEXP_BEG,     "/",          :expr_beg,
+                :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
+                :tREGEXP_END,     "",           :expr_end)
+  end
+
+  def test_yylex_regexp_escape_M_C
+    assert_lex3("/regex\\M-\\C-x/",
+                nil,
+                :tREGEXP_BEG,     "/",              :expr_beg,
+                :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
+                :tREGEXP_END,     "",               :expr_end)
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash
+    refute_lex '/regex\\Mx/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash_eos
+    refute_lex '/regex\\M-/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_dash_eos2
+    refute_lex '/regex\\M-', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_M_bad_eos
+    refute_lex '/regex\\M/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_backslash_slash
+    assert_lex3("/\\//",
+                nil,
+                :tREGEXP_BEG,     "/",   :expr_beg,
+                :tSTRING_CONTENT, "\\/", :expr_beg,
+                :tREGEXP_END,     "",    :expr_end)
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator
+    assert_lex3("%r%blah\\%blah%",
+                nil,
+                :tREGEXP_BEG,     "%r\000",      :expr_beg,
+                :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escaped_delim
+    assert_lex3("%r!blah(?\\!blah)!",
+                nil,
+                :tREGEXP_BEG,     "%r\000",       :expr_beg,
+                :tSTRING_CONTENT, "blah(?!blah)", :expr_beg,
+                :tREGEXP_END,     "",             :expr_end)
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta1
+    assert_lex3("%r{blah\\}blah}",
+                nil,
+                :tREGEXP_BEG,     "%r{",         :expr_beg, # FIX ?!?
+                :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta2
+    assert_lex3("%r/blah\\/blah/",
+                nil,
+                :tREGEXP_BEG,     "%r\000",      :expr_beg,
+                :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_backslash_terminator_meta3
+    assert_lex3("%r/blah\\%blah/",
+                nil,
+                :tREGEXP_BEG,     "%r\000",      :expr_beg,
+                :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_bad_eos
+    refute_lex '/regex\\', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_bs
+    assert_lex3("/regex\\\\regex/",
+                nil,
+                :tREGEXP_BEG,     "/",              :expr_beg,
+                :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
+                :tREGEXP_END,     "",               :expr_end)
+  end
+
+  def test_yylex_regexp_escape_c
+    assert_lex3("/regex\\cxxx/",
+                nil,
+                :tREGEXP_BEG,     "/",           :expr_beg,
+                :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_c_backslash
+    assert_lex3("/regex\\c\\n/",
+                nil,
+                :tREGEXP_BEG,     "/",           :expr_beg,
+                :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_chars
+    assert_lex3("/re\\tge\\nxp/",
+                nil,
+                :tREGEXP_BEG,     "/",            :expr_beg,
+                :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
+                :tREGEXP_END,     "",             :expr_end)
+  end
+
+  def test_yylex_regexp_escape_double_backslash
+    regexp = '/[\\/\\\\]$/'
+    assert_lex3(regexp.dup,
+                nil,
+                :tREGEXP_BEG,     "/",          :expr_beg,
+                :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
+                :tREGEXP_END,     "",           :expr_end)
+  end
+
+  def test_yylex_regexp_escape_hex
+    assert_lex3("/regex\\x61xp/",
+                nil,
+                :tREGEXP_BEG,     "/",            :expr_beg,
+                :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
+                :tREGEXP_END,     "",             :expr_end)
+  end
+
+  def test_yylex_regexp_escape_hex_bad
+    refute_lex '/regex\\xzxp/', :tREGEXP_BEG, "/"
+  end
+
+  def test_yylex_regexp_escape_hex_one
+    assert_lex3("/^[\\xd\\xa]{2}/on",
+                nil,
+                :tREGEXP_BEG,     "/",              :expr_beg,
+                :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
+                :tREGEXP_END,     "on",             :expr_end)
+  end
+
+  def test_yylex_regexp_escape_oct1
+    assert_lex3("/regex\\0xp/",
+                nil,
+                :tREGEXP_BEG,     "/",          :expr_beg,
+                :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
+                :tREGEXP_END,     "",           :expr_end)
+  end
+
+  def test_yylex_regexp_escape_oct2
+    assert_lex3("/regex\\07xp/",
+                nil,
+                :tREGEXP_BEG,     "/",           :expr_beg,
+                :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
+                :tREGEXP_END,     "",            :expr_end)
+  end
+
+  def test_yylex_regexp_escape_oct3
+    assert_lex3("/regex\\10142/",
+                nil,
+                :tREGEXP_BEG,     "/",            :expr_beg,
+                :tSTRING_CONTENT, "regex\\10142", :expr_beg,
+                :tREGEXP_END,     "",             :expr_end)
+  end
+
+  def test_yylex_regexp_escape_return
+    assert_lex3("/regex\\\nregex/",
+                nil,
+                :tREGEXP_BEG,     "/",          :expr_beg,
+                :tSTRING_CONTENT, "regexregex", :expr_beg,
+                :tREGEXP_END,     "",           :expr_end)
+  end
+
+  def test_yylex_regexp_nm
+    assert_lex3("/.*/nm",
+                nil,
+                :tREGEXP_BEG,     "/",  :expr_beg,
+                :tSTRING_CONTENT, ".*", :expr_beg,
+                :tREGEXP_END,     "nm", :expr_end)
+  end
+
+  def test_yylex_rparen
+    assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
+  end
+
+  def test_yylex_rshft
+    assert_lex3("a >> 2",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tRSHFT,      ">>", :expr_beg,
+                :tINTEGER,    2,    :expr_end)
+  end
+
+  def test_yylex_rshft_equals
+    assert_lex3("a >>= 2",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tOP_ASGN,    ">>", :expr_beg,
+                :tINTEGER,    2,    :expr_end)
+  end
+
+  def test_yylex_star
+    assert_lex3("a * ",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tSTAR2,      "*", :expr_beg)
+  end
+
+  def test_yylex_star2
+    assert_lex3("a ** ",
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tPOW,        "**", :expr_beg)
+  end
+
+  def test_yylex_star2_equals
+    assert_lex3("a **= ",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tOP_ASGN,    "**", :expr_beg)
+  end
+
+  def test_yylex_star_arg
+    self.lex_state = :expr_arg
+
+    assert_lex3(" *a",
+                nil,
+                :tSTAR,       "*", :expr_beg,
+                :tIDENTIFIER, "a", :expr_arg)
+  end
+
+  def test_yylex_star_arg_beg
+    self.lex_state = :expr_beg
+
+    assert_lex3("*a",
+                nil,
+                :tSTAR,       "*", :expr_beg,
+                :tIDENTIFIER, "a", :expr_arg)
+  end
+
+  def test_yylex_star_arg_beg_fname
+    self.lex_state = :expr_fname
+
+    assert_lex3("*a",
+                nil,
+                :tSTAR2,      "*", :expr_arg,
+                :tIDENTIFIER, "a", :expr_arg)
+  end
+
+  def test_yylex_star_arg_beg_fname2
+    self.lex_state = :expr_fname
+
+    assert_lex3("*a",
+                nil,
+                :tSTAR2,      "*", :expr_arg,
+                :tIDENTIFIER, "a", :expr_arg)
+  end
+
+  def test_yylex_star_equals
+    assert_lex3("a *= ",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tOP_ASGN, "*", :expr_beg)
+  end
+
+  def test_yylex_string_bad_eos
+    refute_lex('%', :tSTRING_BEG, '%')
+  end
+
+  def test_yylex_string_bad_eos_quote
+    refute_lex('%{nest', :tSTRING_BEG, '%}')
+  end
+
+  def test_yylex_string_double
+    assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_C
+    assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_C_backslash
+    assert_lex3("\"\\C-\\\\\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\034", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_C_escape
+    assert_lex3("\"\\C-\\M-a\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\201", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_C_question
+    assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
+  end
+
+  def test_yylex_string_utf8_simple
+    chr = [0x3024].pack("U")
+
+    assert_lex3('"\u{3024}"',
+                s(:str, chr),
+                :tSTRING, chr, :expr_end)
+  end
+
+  def test_yylex_string_utf8_complex
+    chr = [0x3024].pack("U")
+
+    assert_lex3('"#@a\u{3024}"',
+                s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
+                :tSTRING_BEG,     '"',      :expr_beg,
+                :tSTRING_DVAR,    nil,      :expr_beg,
+                :tSTRING_CONTENT, "@a"+chr, :expr_beg,
+                :tSTRING_END,     '"',      :expr_end)
+  end
+
+  def test_yylex_string_double_escape_M
+    chr = "\341"
+    chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
+
+    assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
+  end
+
+  def test_why_does_ruby_hate_me?
+    assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
+                nil,
+                :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
+  end
+
+  def test_yylex_string_double_escape_M_backslash
+    assert_lex3("\"\\M-\\\\\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\334", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_M_escape
+    assert_lex3("\"\\M-\\C-a\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\201", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_bs1
+    assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_bs2
+    assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_c
+    assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_c_backslash
+    assert_lex3("\"\\c\\\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\034", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_c_escape
+    assert_lex3("\"\\c\\M-a\"",
+                nil,
+                :tSTRING_BEG,     "\"",   :expr_beg,
+                :tSTRING_CONTENT, "\201", :expr_beg,
+                :tSTRING_END,     "\"",   :expr_end)
+  end
+
+  def test_yylex_string_double_escape_c_question
+    assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_chars
+    assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_hex
+    assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_octal
+    assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
+  end
+
+  def test_yylex_string_double_escape_octal_fucked
+    assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
+  end
+
+  def test_yylex_string_double_interp
+    assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
+                nil,
+                :tSTRING_BEG,     "\"",         :expr_beg,
+                :tSTRING_CONTENT, "blah #x a ", :expr_beg,
+                :tSTRING_DVAR,    nil,          :expr_beg,
+                :tSTRING_CONTENT, "@a b ",      :expr_beg,
+                :tSTRING_DVAR,    nil,          :expr_beg,
+                :tSTRING_CONTENT, "$b c ",      :expr_beg,
+                :tSTRING_DBEG,    nil,          :expr_beg,
+                :tSTRING_CONTENT, "3} # ",      :expr_beg,
+                :tSTRING_END,     "\"",         :expr_end)
+  end
+
+  def test_yylex_string_double_pound_dollar_bad
+    skip if ruby18
+
+    assert_lex3('"#$%"', nil,
+
+                :tSTRING_BEG,     "\"",  :expr_beg,
+                :tSTRING_CONTENT, '#$%', :expr_beg,
+                :tSTRING_END,     "\"",  :expr_end)
+  end
+
+  def test_yylex_string_double_nested_curlies
+    assert_lex3("%{nest{one{two}one}nest}",
+                nil,
+                :tSTRING_BEG,     "%}",                    :expr_beg,
+                :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
+                :tSTRING_END,     "}",                     :expr_end)
+  end
+
+  def test_yylex_string_double_no_interp
+    assert_lex3("\"# blah\"",      nil, :tSTRING, "# blah",      :expr_end)
+    assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
+  end
+
+  def test_yylex_string_escape_x_single
+    assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
+  end
+
+  def test_yylex_string_pct_i
+    assert_lex3("%i[s1 s2\ns3]",
+                nil,
+                :tQSYMBOLS_BEG,   "%i[", :expr_beg,
+                :tSTRING_CONTENT, "s1",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s2",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s3",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_pct_I
+    assert_lex3("%I[s1 s2\ns3]",
+                nil,
+                :tSYMBOLS_BEG,    "%I[", :expr_beg,
+                :tSTRING_CONTENT, "s1",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s2",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s3",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_pct_i_extra_space
+    assert_lex3("%i[ s1 s2\ns3 ]",
+                nil,
+                :tQSYMBOLS_BEG,   "%i[", :expr_beg,
+                :tSTRING_CONTENT, "s1",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s2",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s3",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_pct_I_extra_space
+    assert_lex3("%I[ s1 s2\ns3 ]",
+                nil,
+                :tSYMBOLS_BEG,    "%I[", :expr_beg,
+                :tSTRING_CONTENT, "s1",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s2",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s3",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_pct_q
+    assert_lex3("%q[s1 s2]",
+                nil,
+                :tSTRING_BEG,     "%q[",   :expr_beg,
+                :tSTRING_CONTENT, "s1 s2", :expr_beg,
+                :tSTRING_END,     "]",     :expr_end)
+  end
+
+  def test_yylex_string_pct_Q
+    assert_lex3("%Q[s1 s2]",
+                nil,
+                :tSTRING_BEG,     "%Q[",   :expr_beg,
+                :tSTRING_CONTENT, "s1 s2", :expr_beg,
+                :tSTRING_END,     "]",     :expr_end)
+  end
+
+  def test_yylex_string_pct_W
+    assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
+                nil,
+                :tWORDS_BEG,      "%W[", :expr_beg,
+                :tSTRING_CONTENT, "s1",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s2",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s3",  :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_pct_W_bs_nl
+    assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
+                nil,
+                :tWORDS_BEG,      "%W[",  :expr_beg,
+                :tSTRING_CONTENT, "s1",   :expr_beg,
+                :tSPACE,          nil,    :expr_beg,
+                :tSTRING_CONTENT, "\ns2", :expr_beg,
+                :tSPACE,          nil,    :expr_beg,
+                :tSTRING_END,     nil,    :expr_end)
+  end
+
+  def test_yylex_string_pct_angle
+    assert_lex3("%<blah>",
+                nil,
+                :tSTRING_BEG,     "%>",   :expr_beg,
+                :tSTRING_CONTENT, "blah", :expr_beg,
+                :tSTRING_END,     ">",    :expr_end)
+  end
+
+  def test_yylex_string_pct_other
+    assert_lex3("%%blah%",
+                nil,
+                :tSTRING_BEG,     "%%",   :expr_beg,
+                :tSTRING_CONTENT, "blah", :expr_beg,
+                :tSTRING_END,     "%",    :expr_end)
+  end
+
+  def test_yylex_string_pct_w
+    refute_lex("%w[s1 s2 ",
+               :tQWORDS_BEG,     "%w[",
+               :tSTRING_CONTENT, "s1",
+               :tSPACE,          nil,
+               :tSTRING_CONTENT, "s2",
+               :tSPACE,          nil)
+  end
+
+  def test_yylex_string_pct_w_bs_nl
+    assert_lex3("%w[s1 \\\ns2]",
+                nil,
+                :tQWORDS_BEG,     "%w[",  :expr_beg,
+                :tSTRING_CONTENT, "s1",   :expr_beg,
+                :tSPACE,          nil,    :expr_beg,
+                :tSTRING_CONTENT, "\ns2", :expr_beg,
+                :tSPACE,          nil,    :expr_beg,
+                :tSTRING_END,     nil,    :expr_end)
+  end
+
+  def test_yylex_string_pct_w_bs_sp
+    assert_lex3("%w[s\\ 1 s\\ 2]",
+                nil,
+                :tQWORDS_BEG,     "%w[", :expr_beg,
+                :tSTRING_CONTENT, "s 1", :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_CONTENT, "s 2", :expr_beg,
+                :tSPACE,          nil,   :expr_beg,
+                :tSTRING_END,     nil,   :expr_end)
+  end
+
+  def test_yylex_string_single
+    assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
+  end
+
+  def test_yylex_string_single_escape_chars
+    assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
+  end
+
+  def test_yylex_string_single_nl
+    assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
+  end
+
+  def test_yylex_string_single_escaped_quote
+    assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", :expr_end)
+  end
+
+  def test_yylex_symbol
+    assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
+  end
+
+  def test_yylex_symbol_zero_byte__18
+    setup_lexer_class RubyParser::V18
+
+    refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
+  end
+
+  def test_yylex_symbol_zero_byte
+    assert_lex(":\"symbol\0\"", nil,
+                :tSYMBOL,         "symbol\0", :expr_end)
+  end
+
+  def test_yylex_symbol_double
+    assert_lex3(":\"symbol\"",
+                nil,
+                :tSYMBOL,         "symbol", :expr_end)
+  end
+
+  def test_yylex_symbol_double_interp
+    assert_lex3(':"symbol#{1+1}"',
+                nil,
+                :tSYMBEG,         ":",      :expr_fname,
+                :tSTRING_CONTENT, "symbol", :expr_fname,
+                :tSTRING_DBEG,    nil,      :expr_fname,
+                :tSTRING_CONTENT, "1+1}",   :expr_fname, # HUH? this is BS
+                :tSTRING_END,     "\"",     :expr_end)
+  end
+
+  def test_yylex_symbol_single
+    assert_lex3(":'symbol'",
+                nil,
+                :tSYMBOL,         "symbol", :expr_end)
+  end
+
+  def test_yylex_symbol_single_noninterp
+    assert_lex3(':\'symbol#{1+1}\'',
+                nil,
+                :tSYMBOL,   'symbol#{1+1}', :expr_end)
+  end
+
+  def test_yylex_symbol_single_escape_chars
+    assert_lex3(":'s\\tri\\ng'",
+                nil,
+                :tSYMBOL,   "s\\tri\\ng", :expr_end)
+  end
+
+  def test_yylex_string_single_escape_quote_and_backslash
+    assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz", :expr_end)
+  end
+
+  def test_yylex_ternary1
+    assert_lex3("a ? b : c",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tEH,         "?", :expr_value,
+                :tIDENTIFIER, "b", :expr_arg,
+                :tCOLON,      ":", :expr_beg,
+                :tIDENTIFIER, "c", :expr_arg)
+
+    assert_lex3("a ?bb : c", # GAH! MATZ!!!
+                nil,
+                :tIDENTIFIER, "a",  :expr_cmdarg,
+                :tEH,         "?",  :expr_beg,
+                :tIDENTIFIER, "bb", :expr_arg,
+                :tCOLON,      ":",  :expr_beg,
+                :tIDENTIFIER, "c",  :expr_arg)
+
+    assert_lex3("42 ?",
+                nil,
+                :tINTEGER, 42,  :expr_end,
+                :tEH,      "?", :expr_value)
+  end
+
+  def test_yylex_tilde
+    assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
+  end
+
+  def test_yylex_tilde_unary
+    self.lex_state = :expr_fname
+
+    assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
+  end
+
+  def test_yylex_uminus
+    assert_lex3("-blah",
+                nil,
+                :tUMINUS,     "-",    :expr_beg,
+                :tIDENTIFIER, "blah", :expr_arg)
+  end
+
+  def test_yylex_underscore
+    assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
+  end
+
+  def test_yylex_underscore_end
+    assert_lex3("__END__\n",
+                nil,
+                RubyLexer::EOF, RubyLexer::EOF, nil)
+  end
+
+  def test_yylex_uplus
+    assert_lex3("+blah",
+                nil,
+                :tUPLUS,      "+",    :expr_beg,
+                :tIDENTIFIER, "blah", :expr_arg)
+  end
+
+  def test_zbug_float_in_decl
+    assert_lex3("def initialize(u = 0.0, s = 0.0",
+                nil,
+                :kDEF,        "def",        :expr_fname,
+                :tIDENTIFIER, "initialize", :expr_endfn,
+                :tLPAREN2,    "(",          :expr_beg,
+                :tIDENTIFIER, "u",          :expr_arg,
+                :tEQL,        "=",          :expr_beg,
+                :tFLOAT,      0.0,          :expr_end,
+                :tCOMMA,      ",",          :expr_beg,
+                :tIDENTIFIER, "s",          :expr_arg,
+                :tEQL,        "=",          :expr_beg,
+                :tFLOAT,      0.0,          :expr_end)
+  end
+
+  def test_zbug_id_equals
+    assert_lex3("a = 0.0",
+                nil,
+                :tIDENTIFIER, "a", :expr_cmdarg,
+                :tEQL,        "=", :expr_beg,
+                :tFLOAT,      0.0, :expr_end)
+  end
+
+  def test_zbug_no_spaces_in_decl
+    assert_lex3("def initialize(u=0.0,s=0.0",
+                nil,
+                :kDEF,        "def",        :expr_fname,
+                :tIDENTIFIER, "initialize", :expr_endfn,
+                :tLPAREN2,    "(",          :expr_beg,
+                :tIDENTIFIER, "u",          :expr_arg,
+                :tEQL,        "=",          :expr_beg,
+                :tFLOAT,      0.0,          :expr_end,
+                :tCOMMA,      ",",          :expr_beg,
+                :tIDENTIFIER, "s",          :expr_arg,
+                :tEQL,        "=",          :expr_beg,
+                :tFLOAT,      0.0,          :expr_end)
+  end
+
+  def test_pct_w_backslashes
+    ["\t", "\n", "\r", "\v", "\f"].each do |char|
+      next if !RubyLexer::HAS_ENC and char == "\v"
+
+      assert_lex("%w[foo#{char}bar]",
+                 s(:array, s(:str, "foo"), s(:str, "bar")),
+
+                 :tQWORDS_BEG,     "%w[", :expr_beg, 0, 0,
+                 :tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
+                 :tSPACE,          nil,   :expr_beg, 0, 0,
+                 :tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
+                 :tSPACE,          nil,   :expr_beg, 0, 0,
+                 :tSTRING_END,     nil,   :expr_end, 0, 0)
+    end
+  end
+
+  def test_yylex_sym_quoted
+    assert_lex(":'a'",
+               s(:lit, :a),
+
+               :tSYMBOL, "a", :expr_end, 0, 0)
+  end
+
+  def test_yylex_hash_colon
+    assert_lex("{a:1}",
+               s(:hash, s(:lit, :a), s(:lit, 1)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "a", :expr_labeled, 0, 1,
+               :tINTEGER, 1,  :expr_end,      0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_hash_colon_quoted_22
+    setup_lexer_class RubyParser::V22
+
+    assert_lex("{'a':1}",
+               s(:hash, s(:lit, :a), s(:lit, 1)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "a", :expr_labeled, 0, 1,
+               :tINTEGER, 1,  :expr_end,      0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_hash_colon_quoted_symbol_22
+    setup_lexer_class RubyParser::V22
+
+    assert_lex("{'abc': :b}",
+               s(:hash, s(:lit, :abc), s(:lit, :b)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "abc", :expr_labeled, 0, 1,
+               :tSYMBOL, "b",  :expr_end,     0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_hash_colon_double_quoted_symbol_22
+    setup_lexer_class RubyParser::V22
+
+    assert_lex('{"abc": :b}',
+               s(:hash, s(:lit, :abc), s(:lit, :b)),
+
+               :tLBRACE, "{", :expr_beg,      0, 1,
+               :tLABEL,  "abc", :expr_labeled, 0, 1,
+               :tSYMBOL, "b",  :expr_end,     0, 1,
+               :tRCURLY, "}", :expr_endarg,   0, 0)
+  end
+
+  def test_yylex_required_kwarg_no_value_22
+    setup_lexer_class RubyParser::V22
+
+    assert_lex3("def foo a:, b:\nend",
+                nil,
+                :kDEF, "def", :expr_fname,
+                :tIDENTIFIER, "foo", :expr_endfn,
+                :tLABEL, "a",   :expr_labeled,
+                :tCOMMA, ",", :expr_beg,
+                :tLABEL, "b",   :expr_labeled,
+                :kEND, "end", :expr_end)
+  end
+
+  def test_yylex_hash_colon_double_quoted_with_escapes
+    setup_lexer_class RubyParser::V22
+
+    assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
+               nil,
+
+               :tLBRACE, "{", :expr_beg,
+               :tLABEL,  "s\tr\i\ng\\foo'bar", :expr_labeled,
+               :tINTEGER, 1,  :expr_end,
+               :tRCURLY, "}", :expr_endarg)
+  end
+
+  def test_yylex_hash_colon_quoted_with_escapes
+    setup_lexer_class RubyParser::V22
+
+    assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
+               nil,
+
+               :tLBRACE, "{", :expr_beg,
+               :tLABEL,  "s\\tr\\i\\ng\\foo'bar", :expr_labeled,
+               :tINTEGER, 1,  :expr_end,
+               :tRCURLY, "}", :expr_endarg)
+  end
+
+  def test_ruby21_rational_literal
+    setup_lexer_class RubyParser::V21
+
+    assert_lex3("10r",      nil, :tRATIONAL, Rational(10), :expr_end)
+    assert_lex3("0x10r",      nil, :tRATIONAL, Rational(16), :expr_end)
+    assert_lex3("0o10r",      nil, :tRATIONAL, Rational(8), :expr_end)
+    assert_lex3("0or",      nil, :tRATIONAL, Rational(0), :expr_end)
+    assert_lex3("0b10r",      nil, :tRATIONAL, Rational(2), :expr_end)
+    assert_lex3("1.5r",    nil, :tRATIONAL, Rational(15, 10), :expr_end)
+    assert_lex3("15e3r",    nil, :tRATIONAL, Rational(15000), :expr_end)
+    assert_lex3("15e-3r",    nil, :tRATIONAL, Rational(15, 1000), :expr_end)
+    assert_lex3("1.5e3r",    nil, :tRATIONAL, Rational(1500), :expr_end)
+    assert_lex3("1.5e-3r",    nil, :tRATIONAL, Rational(15, 10000), :expr_end)
+
+    assert_lex3("-10r", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tRATIONAL, Rational(10), :expr_end)
+  end
+
+  def test_ruby21_imaginary_literal
+    setup_lexer_class RubyParser::V21
+
+    assert_lex3("1i",       nil, :tIMAGINARY, Complex(0, 1), :expr_end)
+    assert_lex3("0x10i",    nil, :tIMAGINARY, Complex(0, 16), :expr_end)
+    assert_lex3("0o10i",    nil, :tIMAGINARY, Complex(0, 8), :expr_end)
+    assert_lex3("0oi",      nil, :tIMAGINARY, Complex(0, 0), :expr_end)
+    assert_lex3("0b10i",    nil, :tIMAGINARY, Complex(0, 2), :expr_end)
+    assert_lex3("1.5i",    nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
+    assert_lex3("15e3i",    nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
+    assert_lex3("15e-3i",    nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
+    assert_lex3("1.5e3i",    nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
+    assert_lex3("1.5e-3i",    nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
+
+    assert_lex3("-10i", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tIMAGINARY, Complex(0, 10), :expr_end)
+  end
+
+  def test_ruby21_rational_imaginary_literal
+    setup_lexer_class RubyParser::V21
+
+    assert_lex3("1ri",       nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
+    assert_lex3("0x10ri",    nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
+    assert_lex3("0o10ri",    nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
+    assert_lex3("0ori",      nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
+    assert_lex3("0b10ri",    nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
+    assert_lex3("1.5ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
+    assert_lex3("15e3ri",    nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
+    assert_lex3("15e-3ri",    nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
+    assert_lex3("1.5e3ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
+    assert_lex3("1.5e-3ri",    nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
+
+    assert_lex3("-10ri", nil,
+                :tUMINUS_NUM, "-", :expr_beg,
+                :tIMAGINARY, Complex(0, Rational(10)), :expr_end)
+  end
+
+  def test_ruby21_imaginary_literal_with_succeeding_keyword
+    skip "Currently does not tokenize correctly"
+
+    setup_lexer_class RubyParser::V21
+
+    assert_lex3("1if", nil,
+                :tINTEGER, 1, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1rif", nil,
+                :tRATIONAL, Rational(1), :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1.0if", nil,
+                :tFLOAT, 1.0, :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+    assert_lex3("1.0rif", nil,
+                :tRATIONAL, Rational("1.0"), :expr_end,
+                :kIF_MOD, "if", :expr_beg)
+
+    flunk
+  end
+end