parser 0.9.alpha1 → 0.9.0

data/test/racc_coverage_helper.rb

@@ -0,0 +1,130 @@
+require 'racc/grammarfileparser'
+
+# Unfortunately, Ruby's Coverage module ignores module_eval statements,
+# which Racc uses to map `parser.y` locations in the generated
+# `parser.rb`.
+module RaccCoverage
+  @coverage  = {}
+  @base_path = nil
+  @trace     = nil
+
+  def self.start(parsers, base_path)
+    @base_path = base_path
+
+    parsers.each do |parser|
+      @coverage[parser] = extract_interesting_lines(parser, base_path)
+    end
+
+    @trace = TracePoint.trace(:line) do |trace|
+      lineno = trace.lineno - 1
+
+      if (line_coverage = @coverage[trace.path])
+        if line_coverage[lineno]
+          line_coverage[lineno] += 1
+        end
+      end
+    end
+  end
+
+  def self.stop
+    @trace.disable
+  end
+
+  # Ruby's TracePoint#lineno will point only on "interesting" lines,
+  # i.e.: only code (no comments or empty lines), no `end` keywords,
+  # and for multi-line statements, only the first line of the statement.
+  #
+  # This method implements a very dumb Ruby parser, which skips empty lines
+  # or lines with just comments, `end` keywords, and correctly handles
+  # multi-line statements of the following form:
+  #
+  #  * All lines of the statement except the last must end with `,`, `.` or `(`.
+  #
+  # Coverage can be disabled for code regions with annotations :nocov: and :cov:.
+  #
+  # Also, for best results, all actions should be delimited by at least
+  # one non-action line.
+  #
+  def self.extract_interesting_lines(parser, base_path)
+    grammar_source = File.join(@base_path, parser)
+    grammar_file   = Racc::GrammarFileParser.parse_file(grammar_source)
+
+    ruby_sources = [
+      # Header and footer aren't passed through module_eval
+      # in Racc-generated file, so the location info is lost.
+      *grammar_file.params.inner,
+    ].compact
+
+    grammar_file.grammar.each_rule do |rule|
+      source = rule.action.source
+      next if source.nil?
+
+      ruby_sources << source
+    end
+
+    lines = []
+
+    ruby_sources.each do |source|
+      first_line = source.lineno
+
+      state = :first_line
+
+      source.text.each_line.with_index do |line, index|
+        line = line.strip
+
+        continues = line.end_with?(',')   ||
+                      line.end_with?('(') ||
+                      line.end_with?('.')
+
+        case state
+        when :first_line
+          if line =~ /:nocov/
+            state = :nocov
+            next
+          elsif line.empty?   ||
+                  line == 'end' ||
+                  line.start_with?('#')
+            next
+          elsif continues
+            state = :mid_line
+          end
+
+          lines[first_line + index - 1] = 0
+
+        when :mid_line
+          unless continues
+            state = :first_line
+          end
+
+        when :nocov
+          if line =~ /:cov:/
+            state = :first_line
+          end
+        end
+      end
+    end
+
+    lines
+  end
+
+  def self.result
+    result =
+      @coverage.map do |parser, coverage|
+        [File.join(@base_path, parser), coverage]
+      end
+
+    Hash[result]
+  end
+end
+
+class << SimpleCov
+  def result_with_racc_coverage
+    @result ||= SimpleCov::Result.new(
+                  Coverage.result.merge(RaccCoverage.result))
+
+    result_without_racc_coverage
+  end
+
+  alias result_without_racc_coverage result
+  alias result result_with_racc_coverage
+end

data/test/test_diagnostic.rb

@@ -0,0 +1,47 @@
+require 'helper'
+
+class TestDiagnostic < MiniTest::Unit::TestCase
+  def setup
+    @buffer = Parser::Source::Buffer.new('(string)')
+    @buffer.source = "if (this is some bad code + bugs)"
+
+    @range1 = Parser::Source::Range.new(@buffer, 0, 1) # if
+    @range2 = Parser::Source::Range.new(@buffer, 4, 7) # this
+  end
+
+  def test_verifies_levels
+    assert_raises ArgumentError, /level/ do
+      Parser::Diagnostic.new(:foobar, "foo", @range1)
+    end
+  end
+
+  def test_freezes
+    string     = "foo"
+    highlights = [@range2]
+
+    diag = Parser::Diagnostic.new(:error, string, @range1, highlights)
+    assert diag.frozen?
+    assert diag.message.frozen?
+    assert diag.highlights.frozen?
+
+    refute string.frozen?
+    refute highlights.frozen?
+  end
+
+  def test_render
+    location = Parser::Source::Range.new(@buffer, 26, 26)
+
+    highlights = [
+      Parser::Source::Range.new(@buffer, 21, 24),
+      Parser::Source::Range.new(@buffer, 28, 31)
+    ]
+
+    diag  = Parser::Diagnostic.new(:error, "code far too bad",
+                                   location, highlights)
+    assert_equal([
+      "(string):1:27: error: code far too bad",
+      "if (this is some bad code + bugs)",
+      "                     ~~~~ ^ ~~~~ "
+    ], diag.render)
+  end
+end

data/test/test_diagnostic_engine.rb

@@ -0,0 +1,58 @@
+require 'helper'
+
+class TestDiagnosticEngine < MiniTest::Unit::TestCase
+  def setup
+    @buffer  = Parser::Source::Buffer.new('(source)')
+    @buffer.source = 'foobar'
+
+    @engine = Parser::Diagnostic::Engine.new
+
+    @queue  = []
+    @engine.consumer = lambda { |diag| @queue << diag }
+  end
+
+  def test_process_warnings
+    warn = Parser::Diagnostic.new(:warning, "foo", @buffer, 1..2)
+    @engine.process(warn)
+
+    assert_equal [warn], @queue
+  end
+
+  def test_ignore_warnings
+    @engine.ignore_warnings = true
+
+    warn = Parser::Diagnostic.new(:warning, "foo", @buffer, 1..2)
+    @engine.process(warn)
+
+    assert_equal [], @queue
+  end
+
+  def test_all_errors_are_fatal
+    @engine.all_errors_are_fatal = true
+
+    error = Parser::Diagnostic.new(:error, "foo", @buffer, 1..2)
+
+    assert_raises Parser::SyntaxError do
+      @engine.process(error)
+    end
+
+    assert_equal [error], @queue
+  end
+
+  def test_all_errors_are_collected
+    error = Parser::Diagnostic.new(:error, "foo", @buffer, 1..2)
+    @engine.process(error)
+
+    assert_equal [error], @queue
+  end
+
+  def test_fatal_error
+    fatal = Parser::Diagnostic.new(:fatal, "foo", @buffer, 1..2)
+
+    assert_raises Parser::SyntaxError do
+      @engine.process(fatal)
+    end
+
+    assert_equal [fatal], @queue
+  end
+end

data/test/test_lexer.rb

@@ -1,28 +1,38 @@
 # encoding: ascii-8bit
 
-require 'minitest/autorun'
-require 'parser/lexer'
+require 'helper'
 
 class TestLexer < MiniTest::Unit::TestCase
-  def setup_lexer version
+  def setup_lexer(version)
     @lex = Parser::Lexer.new(version)
+
+    @lex.diagnostics = Parser::Diagnostic::Engine.new
+    @lex.diagnostics.all_errors_are_fatal = true
+    # @lex.diagnostics.consumer = lambda { |diag| $stderr.puts "", diag.render }
   end
 
   def setup
     setup_lexer 18
   end
 
-  def util_bad_token s, *args
+  #
+  # Additional matchers
+  #
+
+  def util_bad_token(s, *args)
     assert_raises Parser::SyntaxError do
-      util_lex_token s, *args
+      util_lex_token(s, *args)
     end
   end
 
-  def util_escape expected, input
+  def util_escape(expected, input)
+    source_buffer = Parser::Source::Buffer.new('(util_escape)')
+    source_buffer.source = "%Q[\\#{input}]"
+
     @lex.reset
-    @lex.source = "%Q[\\#{input}]"
+    @lex.source_buffer = source_buffer
 
-    lex_token, lex_value = @lex.advance
+    lex_token, (lex_value, *) = @lex.advance
 
     if lex_value.respond_to?(:force_encoding)
       lex_value.force_encoding('ASCII-8BIT')
@@ -30,47 +40,44 @@ class TestLexer < MiniTest::Unit::TestCase
 
     assert_equal [:tSTRING, expected],
                  [lex_token, lex_value],
-                 @lex.source
+                 source_buffer.source
   end
 
-  def util_escape_bad input
+  def util_escape_bad(input)
     assert_raises Parser::SyntaxError do
       @lex.state = :expr_beg
       util_lex_token "%Q[\\#{input}]"
     end
   end
 
-  def util_lex_fname name, type, end_state = :expr_end
-    util_lex_token("def #{name} ", :kDEF, "def", type, name)
+  def util_lex_fname(name, type, end_state = :expr_end)
+    util_lex_token("def #{name} ", :kDEF, 'def', type, name)
 
     assert_equal end_state, @lex.state
   end
 
-  def util_lex_token input, *args
+  def util_lex_token(input, *args)
+    source_buffer = Parser::Source::Buffer.new('(util_lex_token)')
+    source_buffer.source = input
+
     @lex.reset(false)
-    @lex.source = input
+    @lex.source_buffer = source_buffer
 
     until args.empty? do
       token, value = args.shift(2)
 
-      lex_token, lex_value = @lex.advance
-      assert lex_token, "no more tokens"
+      lex_token, (lex_value, *) = @lex.advance
+      assert lex_token, 'no more tokens'
       assert_equal [token, value], [lex_token, lex_value], input
     end
 
-    lex_token, lex_value = @lex.advance
+    lex_token, (lex_value, *) = @lex.advance
     refute lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
   end
 
-  def test_advance
-    @lex.source = "blah"
-
-    token, = @lex.advance
-    assert token # blah
-
-    token, = @lex.advance
-    refute token # nada
-  end
+  #
+  # Tests
+  #
 
   def test_read_escape
     util_escape "\\",   "\\"
@@ -121,19 +128,23 @@ class TestLexer < MiniTest::Unit::TestCase
   end
 
   def test_read_escape_unicode__19
-    util_escape "\xc4\xa3", 'u0123'
+    if RUBY_VERSION >= '1.9'
+      util_escape "\xc4\xa3", 'u0123'
 
-    util_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", 'u{123 f0 B123}'
+      util_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", 'u{123 f0 B123}'
+    end
   end
 
   def test_read_escape_unicode_bad__19
-    util_escape_bad 'u123'
-    util_escape_bad 'u{}'
-    util_escape_bad 'u{123 f0h}'
-    util_escape_bad 'u{123 f0'
+    if RUBY_VERSION >= '1.9'
+      util_escape_bad 'u123'
+      util_escape_bad 'u{}'
+      util_escape_bad 'u{123 f0h}'
+      util_escape_bad 'u{123 f0'
+    end
   end
 
-  def test_yylex_ambiguous_uminus
+  def test_ambiguous_uminus
     util_lex_token("m -3",
                    :tIDENTIFIER, "m",
                    :tUMINUS_NUM, "-",
@@ -141,30 +152,30 @@ class TestLexer < MiniTest::Unit::TestCase
     # TODO: verify warning
   end
 
-  def test_yylex_ambiguous_uplus
+  def test_ambiguous_uplus
     util_lex_token("m +3",
                    :tIDENTIFIER, "m",
                    :tINTEGER, 3)
     # TODO: verify warning
   end
 
-  def test_yylex_and
+  def test_and
     util_lex_token "&", :tAMPER, "&"
   end
 
-  def test_yylex_and2
+  def test_and2
     @lex.state = :expr_end
 
     util_lex_token "&&", :tANDOP, "&&"
   end
 
-  def test_yylex_and2_equals
+  def test_and2_equals
     @lex.state = :expr_end
 
     util_lex_token "&&=", :tOP_ASGN, "&&"
   end
 
-  def test_yylex_and_arg
+  def test_and_arg
     @lex.state = :expr_arg
 
     util_lex_token(" &y",
@@ -172,13 +183,13 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "y")
   end
 
-  def test_yylex_and_equals
+  def test_and_equals
     @lex.state = :expr_end
 
     util_lex_token "&=", :tOP_ASGN, "&"
   end
 
-  def test_yylex_and_expr
+  def test_and_expr
     @lex.state = :expr_arg
 
     util_lex_token("x & y",
@@ -187,22 +198,22 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "y")
   end
 
-  def test_yylex_and_meth
+  def test_and_meth
     util_lex_fname "&", :tAMPER2
   end
 
-  def test_yylex_assoc
+  def test_assoc
     util_lex_token "=>", :tASSOC, "=>"
   end
 
-  def test_yylex_label__18
+  def test_label__18
     util_lex_token("{a:b",
                    :tLBRACE,     "{",
                    :tIDENTIFIER, "a",
                    :tSYMBOL,     "b")
   end
 
-  def test_yylex_label_in_params__18
+  def test_label_in_params__18
     util_lex_token("foo(a:b",
                    :tIDENTIFIER, "foo",
                    :tLPAREN2,    "(",
@@ -210,7 +221,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSYMBOL,     "b")
   end
 
-  def test_yylex_label__19
+  def test_label__19
     setup_lexer 19
 
     util_lex_token("{a:b",
@@ -219,7 +230,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "b")
   end
 
-  def test_yylex_label_in_params__19
+  def test_label_in_params__19
     setup_lexer 19
 
     util_lex_token("foo(a:b",
@@ -229,7 +240,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "b")
   end
 
-  def test_yylex_command_start__19
+  def test_command_start__19
     setup_lexer 19
 
     %w[case elsif for in until when while
@@ -244,7 +255,7 @@ class TestLexer < MiniTest::Unit::TestCase
     end
   end
 
-  def test_yylex_mod_not_command_start__19
+  def test_mod_not_command_start__19
     setup_lexer 19
 
     %w[if unless while until rescue].each do |keyword|
@@ -258,43 +269,43 @@ class TestLexer < MiniTest::Unit::TestCase
     end
   end
 
-  def test_yylex_back_ref
+  def test_back_ref
     util_lex_token("[$&, $`, $', $+]",
                    :tLBRACK,   "[",
-                   :tBACK_REF, :"&", :tCOMMA, ",",
-                   :tBACK_REF, :"`", :tCOMMA, ",",
-                   :tBACK_REF, :"'", :tCOMMA, ",",
-                   :tBACK_REF, :"+",
+                   :tBACK_REF, "$&", :tCOMMA, ",",
+                   :tBACK_REF, "$`", :tCOMMA, ",",
+                   :tBACK_REF, "$'", :tCOMMA, ",",
+                   :tBACK_REF, "$+",
                    :tRBRACK,   "]")
   end
 
-  def test_yylex_backslash
+  def test_backslash
     util_lex_token("1 \\\n+ 2",
                    :tINTEGER, 1,
                    :tPLUS, "+",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_backslash_bad
+  def test_backslash_bad
     util_bad_token("1 \\ + 2",
                    :tINTEGER, 1)
   end
 
-  def test_yylex_backtick
+  def test_backtick
     util_lex_token("`ls`",
                    :tXSTRING_BEG, "`",
                    :tSTRING_CONTENT, "ls",
                    :tSTRING_END, "`")
   end
 
-  def test_yylex_backtick_cmdarg
+  def test_backtick_cmdarg
     @lex.state = :expr_dot
     util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
 
     assert_equal :expr_arg, @lex.state
   end
 
-  def test_yylex_backtick_dot
+  def test_backtick_dot
     @lex.state = :expr_dot
     util_lex_token("a.`(3)",
                    :tIDENTIFIER, "a",
@@ -305,54 +316,64 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRPAREN, ")")
   end
 
-  def test_yylex_backtick_method
+  def test_backtick_method
     @lex.state = :expr_fname
     util_lex_token("`", :tBACK_REF2, "`")
     assert_equal :expr_end, @lex.state
   end
 
-  def test_yylex_bad_char
+  def test_bad_char
     util_bad_token(" \010 ")
   end
 
-  def test_yylex_bang
+  def test_bang
     util_lex_token "!", :tBANG, "!"
   end
 
-  def test_yylex_bang_equals
+  def test_bang_equals
     util_lex_token "!=", :tNEQ, "!="
   end
 
-  def test_yylex_bang_tilde
+  def test_bang_tilde
     util_lex_token "!~", :tNMATCH, "!~"
   end
 
-  def test_yylex_carat
+  def test_carat
     util_lex_token "^", :tCARET, "^"
   end
 
-  def test_yylex_carat_equals
+  def test_carat_equals
     util_lex_token "^=", :tOP_ASGN, "^"
   end
 
-  def test_yylex_colon2
+  def test_colon2
     util_lex_token("A::B",
                    :tCONSTANT, "A",
                    :tCOLON2,   "::",
                    :tCONSTANT, "B")
+
+    @lex.state = :expr_arg
+    util_lex_token("::Array",
+                   :tCOLON2, "::",
+                   :tCONSTANT, "Array")
   end
 
-  def test_yylex_colon3
+  def test_colon3
     util_lex_token("::Array",
                    :tCOLON3, "::",
                    :tCONSTANT, "Array")
+
+    @lex.state = :expr_arg
+    util_lex_token(" ::Array",
+                   :tCOLON3, "::",
+                   :tCONSTANT, "Array")
   end
 
-  def test_yylex_comma
+  def test_comma
     util_lex_token ",", :tCOMMA, ","
   end
 
-  def test_yylex_comment
+  def test_comment
     util_lex_token("1 # one\n# two\n2",
                    :tINTEGER, 1,
                    :tNL, nil,
@@ -360,18 +381,23 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal "# one\n# two\n", @lex.comments
   end
 
-  def test_yylex_comment_begin
+  def test_comment_expr_beg
+    util_lex_token("{#1\n}",
+                   :tLBRACE, "{",
+                   :tRCURLY, "}")
+  end
+
+  def test_comment_begin
     util_lex_token("=begin\nblah\nblah\n=end\n42",
                    :tINTEGER, 42)
     assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
   end
 
-  def test_yylex_comment_begin_bad
+  def test_comment_begin_bad
     util_bad_token("=begin\nblah\nblah\n")
-    assert_equal "", @lex.comments
   end
 
-  def test_yylex_comment_begin_not_comment
+  def test_comment_begin_not_comment
     util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
                    :tIDENTIFIER, "beginfoo",
                    :tEQL,          "=",
@@ -383,54 +409,54 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tIDENTIFIER, "beginfoo")
   end
 
-  def test_yylex_comment_begin_space
+  def test_comment_begin_space
     util_lex_token("=begin blah\nblah\n=end\n")
     assert_equal "=begin blah\nblah\n=end\n", @lex.comments
   end
 
-  def test_yylex_comment_end_space_and_text
+  def test_comment_end_space_and_text
     util_lex_token("=begin blah\nblah\n=end blab\n")
     assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
   end
 
-  def test_yylex_comment_eos
+  def test_comment_eos
     util_lex_token("# comment")
   end
 
-  def test_yylex_constant
+  def test_constant
     util_lex_token("ArgumentError",
                    :tCONSTANT, "ArgumentError")
   end
 
-  def test_yylex_constant_semi
+  def test_constant_semi
     util_lex_token("ArgumentError;",
                    :tCONSTANT, "ArgumentError",
                    :tSEMI, ";")
   end
 
-  def test_yylex_cvar
+  def test_cvar
     util_lex_token "@@blah", :tCVAR, "@@blah"
   end
 
-  def test_yylex_cvar_bad
+  def test_cvar_bad
     util_bad_token "@@1"
   end
 
-  def test_yylex_div
+  def test_div
     util_lex_token("a / 2",
                    :tIDENTIFIER, "a",
                    :tDIVIDE, "/",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_div_equals
+  def test_div_equals
     util_lex_token("a /= 2",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, "/",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_do
+  def test_do
     util_lex_token("x do 42 end",
                    :tIDENTIFIER, "x",
                    :kDO, "do",
@@ -438,7 +464,17 @@ class TestLexer < MiniTest::Unit::TestCase
                    :kEND, "end")
   end
 
-  def test_yylex_do_block
+  def test_do_cond
+    @lex.cond.push(true)
+
+    util_lex_token("x do 42 end",
+                   :tIDENTIFIER, "x",
+                   :kDO_COND, "do",
+                   :tINTEGER, 42,
+                   :kEND, "end")
+  end
+
+  def test_do_block
     @lex.state = :expr_endarg
 
     util_lex_token("do 42 end",
@@ -448,7 +484,7 @@ class TestLexer < MiniTest::Unit::TestCase
   end
 
   # TODO
-  # def test_yylex_do_cond
+  # def test_do_cond
   #   @lex.cond.push true
 
   #   util_lex_token("x do 42 end",
@@ -458,156 +494,149 @@ class TestLexer < MiniTest::Unit::TestCase
   #                  :kEND, "end")
   # end
 
-  def test_yylex_dot
+  def test_dot
     util_lex_token ".", :tDOT, "."
   end
 
-  def test_yylex_dot2
+  def test_dot2
     util_lex_token "..", :tDOT2, ".."
   end
 
-  def test_yylex_dot3
+  def test_dot3
     util_lex_token "...", :tDOT3, "..."
   end
 
-  def test_yylex_equals
+  def test_equals
     util_lex_token "=", :tEQL, "="
   end
 
-  def test_yylex_equals2
+  def test_equals2
     util_lex_token "==", :tEQ, "=="
   end
 
-  def test_yylex_equals3
+  def test_equals3
     util_lex_token "===", :tEQQ, "==="
   end
 
-  def test_yylex_equals_tilde
+  def test_equals_tilde
     util_lex_token "=~", :tMATCH, "=~"
   end
 
-  def test_yylex_float
+  def test_float
     util_lex_token "1.0", :tFLOAT, 1.0
   end
 
-  def test_yylex_float_bad_no_underscores
+  def test_float_bad_no_underscores
     util_bad_token "1__0.0"
   end
 
-  def test_yylex_float_bad_no_zero_leading
+  def test_float_bad_no_zero_leading
     util_bad_token ".0"
   end
 
-  def test_yylex_float_bad_trailing_underscore
+  def test_float_bad_trailing_underscore
     util_bad_token "123_.0"
   end
 
-  def test_yylex_float_call
+  def test_float_call
     util_lex_token("1.0.to_s",
                    :tFLOAT, 1.0,
                    :tDOT, ".",
                    :tIDENTIFIER, "to_s")
   end
 
-  def test_yylex_float_dot_E
+  def test_float_dot_E
     util_lex_token "1.0E10", :tFLOAT, 1.0e10
   end
 
-  def test_yylex_float_dot_E_neg
+  def test_float_dot_E_neg
     util_lex_token("-1.0E10",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1.0e10)
   end
 
-  def test_yylex_float_dot_e
+  def test_float_dot_e
     util_lex_token "1.0e10", :tFLOAT, 1.0e10
   end
 
-  def test_yylex_float_dot_e_neg
+  def test_float_dot_e_neg
     util_lex_token("-1.0e10",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1.0e10)
   end
 
-  def test_yylex_float_e
+  def test_float_e
     util_lex_token "1e10", :tFLOAT, 1e10
   end
 
-  def test_yylex_float_e_bad_trailing_underscore
+  def test_float_e_bad_trailing_underscore
     util_bad_token "123_e10"
   end
 
-  def test_yylex_float_e_minus
+  def test_float_e_minus
     util_lex_token "1e-10", :tFLOAT, 1e-10
   end
 
-  def test_yylex_float_e_neg
+  def test_float_e_neg
     util_lex_token("-1e10",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1e10)
   end
 
-  def test_yylex_float_e_neg_minus
+  def test_float_e_neg_minus
     util_lex_token("-1e-10",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1e-10)
   end
 
-  def test_yylex_float_e_neg_plus
+  def test_float_e_neg_plus
     util_lex_token("-1e+10",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1e10)
   end
 
-  def test_yylex_float_e_plus
+  def test_float_e_plus
     util_lex_token "1e+10", :tFLOAT, 1e10
   end
 
-  def test_yylex_float_e_zero
+  def test_float_e_zero
     util_lex_token "0e0", :tFLOAT, 0e0
   end
 
-  def test_yylex_float_neg
+  def test_float_neg
     util_lex_token("-1.0",
                    :tUMINUS_NUM, "-",
                    :tFLOAT, 1.0)
   end
 
-  def test_yylex_ge
+  def test_ge
     util_lex_token("a >= 2",
                    :tIDENTIFIER, "a",
                    :tGEQ, ">=",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_global
+  def test_global
     util_lex_token("$blah", :tGVAR, "$blah")
   end
 
-  def test_yylex_global_backref
-    @lex.state = :expr_fname
-    util_lex_token("$`", :tGVAR, "$`")
+  def test_global_backref
+    util_lex_token("$`", :tBACK_REF, "$`")
   end
 
-  def test_yylex_global_dash_nothing
+  def test_global_dash_nothing
     util_lex_token("$- ", :tGVAR, "$-")
   end
 
-  def test_yylex_global_dash_something
+  def test_global_dash_something
     util_lex_token("$-x", :tGVAR, "$-x")
   end
 
-  def test_yylex_global_number
-    @lex.state = :expr_fname
-    util_lex_token("$1", :tGVAR, "$1")
-  end
-
-  def test_yylex_global_number_big
-    @lex.state = :expr_fname
-    util_lex_token("$1234", :tGVAR, "$1234")
+  def test_global_number
+    util_lex_token("$10", :tNTH_REF, 10)
   end
 
-  def test_yylex_global_other
+  def test_global_other
     util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
                    :tLBRACK, "[",
                    :tGVAR,   "$~",  :tCOMMA, ",",
@@ -629,28 +658,28 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRBRACK, "]")
   end
 
-  def test_yylex_global_underscore
+  def test_global_underscore
     util_lex_token("$_",
                    :tGVAR,     "$_")
   end
 
-  def test_yylex_global_wierd
+  def test_global_wierd
     util_lex_token("$__blah",
                    :tGVAR,     "$__blah")
   end
 
-  def test_yylex_global_zero
+  def test_global_zero
     util_lex_token("$0", :tGVAR, "$0")
   end
 
-  def test_yylex_gt
+  def test_gt
     util_lex_token("a > 2",
                    :tIDENTIFIER, "a",
                    :tGT, ">",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_heredoc_backtick
+  def test_heredoc_backtick
     util_lex_token("a = <<`EOF`\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -660,7 +689,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_double
+  def test_heredoc_double
     util_lex_token("a = <<\"EOF\"\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -670,7 +699,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_double_dash
+  def test_heredoc_double_dash
     util_lex_token("a = <<-\"EOF\"\n  blah blah\n  EOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -680,21 +709,21 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_double_eos
+  def test_heredoc_double_eos
     util_bad_token("a = <<\"EOF\"\nblah",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"")
   end
 
-  def test_yylex_heredoc_double_eos_nl
+  def test_heredoc_double_eos_nl
     util_bad_token("a = <<\"EOF\"\nblah\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"")
   end
 
-  def test_yylex_heredoc_double_interp
+  def test_heredoc_double_interp
     util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -714,7 +743,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_empty
+  def test_heredoc_empty
     util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_DBEG,    "\#{",
@@ -726,7 +755,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_none
+  def test_heredoc_none
     util_lex_token("a = <<EOF\nblah\nblah\nEOF",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -737,7 +766,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_none_dash
+  def test_heredoc_none_dash
     util_lex_token("a = <<-EOF\nblah\nblah\n  EOF",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -748,7 +777,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_single
+  def test_heredoc_single
     util_lex_token("a = <<'EOF'\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -758,14 +787,14 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_heredoc_single_bad_eos_body
+  def test_heredoc_single_bad_eos_body
     util_bad_token("a = <<'EOF'\nblah",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
                    :tSTRING_BEG,     "'")
   end
 
-  def test_yylex_heredoc_single_dash
+  def test_heredoc_single_dash
     util_lex_token("a = <<-'EOF'\n  blah blah\n  EOF\n",
                    :tIDENTIFIER,     "a",
                    :tEQL,              "=",
@@ -775,59 +804,64 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
-  def test_yylex_identifier
+  def test_identifier
     util_lex_token("identifier", :tIDENTIFIER, "identifier")
   end
 
-  def test_yylex_identifier_bang
-    util_lex_token("identifier!", :tFID, "identifier!")
+  def test_identifier_bang
+    util_lex_token("identifier!",
+                   :tFID, "identifier!")
+
+    util_lex_token("identifier!=",
+                   :tFID, "identifier",
+                   :tNEQ, "!=")
   end
 
-  def test_yylex_identifier_cmp
+  def test_identifier_cmp
     util_lex_fname "<=>", :tCMP
   end
 
-  def test_yylex_identifier_def
+  def test_identifier_def
     util_lex_fname "identifier", :tIDENTIFIER, :expr_end
   end
 
-  def test_yylex_identifier_eh
+  def test_identifier_eh
     util_lex_token("identifier?", :tFID, "identifier?")
   end
 
-  def test_yylex_identifier_equals_arrow
+  def test_identifier_equals_arrow
     util_lex_token(":blah==>",
                    :tSYMBOL, "blah=",
                    :tASSOC, "=>")
   end
 
-  def test_yylex_identifier_equals3
+  def test_identifier_equals3
     util_lex_token(":a===b",
                    :tSYMBOL, "a",
                    :tEQQ, "===",
                    :tIDENTIFIER, "b")
   end
 
-  def test_yylex_identifier_equals_equals_arrow
+  def test_identifier_equals_equals_arrow
     util_lex_token(":a==>b",
                    :tSYMBOL, "a=",
                    :tASSOC, "=>",
                    :tIDENTIFIER, "b")
   end
 
-  def test_yylex_identifier_equals_caret
+  def test_identifier_equals_caret
     util_lex_fname "^", :tCARET
   end
 
-  def test_yylex_identifier_equals_def
+  def test_identifier_equals_def
     util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
   end
 
-  def test_yylex_identifier_equals_def2
+  def test_identifier_equals_def2
     util_lex_fname "==", :tEQ
   end
 
-  def test_yylex_identifier_equals_expr
+  def test_identifier_equals_expr
     @lex.state = :expr_dot
     util_lex_token("y = arg",
                    :tIDENTIFIER, "y",
@@ -837,193 +871,193 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_arg, @lex.state
   end
 
-  def test_yylex_identifier_equals_or
+  def test_identifier_equals_or
     util_lex_fname "|", :tPIPE
   end
 
-  def test_yylex_identifier_equals_slash
+  def test_identifier_equals_slash
     util_lex_fname "/", :tDIVIDE
   end
 
-  def test_yylex_identifier_equals_tilde
+  def test_identifier_equals_tilde
     @lex.state = :expr_fname # can only set via parser's defs
     util_lex_token("identifier=~",
                    :tIDENTIFIER, "identifier",
                    :tMATCH, "=~")
   end
 
-  def test_yylex_identifier_gt
+  def test_identifier_gt
     util_lex_fname ">", :tGT
   end
 
-  def test_yylex_identifier_le
+  def test_identifier_le
     util_lex_fname "<=", :tLEQ
   end
 
-  def test_yylex_identifier_lt
+  def test_identifier_lt
     util_lex_fname "<", :tLT
   end
 
-  def test_yylex_identifier_tilde
+  def test_identifier_tilde
     util_lex_fname "~", :tTILDE
   end
 
-  def test_yylex_index
+  def test_index
     util_lex_fname "[]", :tAREF
   end
 
-  def test_yylex_index_equals
+  def test_index_equals
     util_lex_fname "[]=", :tASET
   end
 
-  def test_yylex_integer
+  def test_integer
     util_lex_token "42", :tINTEGER, 42
   end
 
-  def test_yylex_integer_bin
+  def test_integer_bin
     util_lex_token "0b101010", :tINTEGER, 42
   end
 
-  def test_yylex_integer_bin_bad_none
+  def test_integer_bin_bad_none
     util_bad_token "0b "
   end
 
-  def test_yylex_integer_bin_bad_underscores
+  def test_integer_bin_bad_underscores
     util_bad_token "0b10__01"
   end
 
-  def test_yylex_integer_dec
+  def test_integer_dec
     util_lex_token "42", :tINTEGER, 42
   end
 
-  def test_yylex_integer_dec_bad_underscores
+  def test_integer_dec_bad_underscores
     util_bad_token "42__24"
   end
 
-  def test_yylex_integer_dec_d
+  def test_integer_dec_d
     util_lex_token "0d42", :tINTEGER, 42
   end
 
-  def test_yylex_integer_dec_d_bad_none
+  def test_integer_dec_d_bad_none
     util_bad_token "0d"
   end
 
-  def test_yylex_integer_dec_d_bad_underscores
+  def test_integer_dec_d_bad_underscores
     util_bad_token "0d42__24"
   end
 
-  def test_yylex_question_eh_a__18
+  def test_question_eh_a__18
     setup_lexer 18
 
     util_lex_token "?a", :tINTEGER, 97
   end
 
-  def test_yylex_question_eh_a__19
+  def test_question_eh_a__19
     setup_lexer 19
 
     util_lex_token '?a', :tSTRING, "a"
   end
 
-  def test_yylex_question_eh_escape_M_escape_C__18
+  def test_question_eh_escape_M_escape_C__18
     setup_lexer 18
 
     util_lex_token '?\M-\C-a', :tINTEGER, 129
   end
 
-  def test_yylex_question_eh_escape_M_escape_C__19
+  def test_question_eh_escape_M_escape_C__19
     setup_lexer 19
 
     util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
   end
 
-  def test_yylex_integer_hex
+  def test_integer_hex
     util_lex_token "0x2a", :tINTEGER, 42
   end
 
-  def test_yylex_integer_hex_bad_none
+  def test_integer_hex_bad_none
     util_bad_token "0x "
   end
 
-  def test_yylex_integer_hex_bad_underscores
+  def test_integer_hex_bad_underscores
     util_bad_token "0xab__cd"
   end
 
-  def test_yylex_integer_oct
+  def test_integer_oct
     util_lex_token "052", :tINTEGER, 42
   end
 
-  def test_yylex_integer_oct_bad_range
+  def test_integer_oct_bad_range
     util_bad_token "08"
   end
 
-  def test_yylex_integer_oct_bad_underscores
+  def test_integer_oct_bad_underscores
     util_bad_token "01__23"
   end
 
-  def test_yylex_integer_oct_O
+  def test_integer_oct_O
     util_lex_token "0O52", :tINTEGER, 42
   end
 
-  def test_yylex_integer_oct_O_bad_range
-    util_bad_token "0O8"
+  def test_integer_oct_O_bad_range
+    util_bad_token "0O1238"
   end
 
-  def test_yylex_integer_oct_O_bad_underscores
+  def test_integer_oct_O_bad_underscores
     util_bad_token "0O1__23"
   end
 
-  def test_yylex_integer_oct_O_not_bad_none
+  def test_integer_oct_O_not_bad_none
     util_lex_token "0O ", :tINTEGER, 0
   end
 
-  def test_yylex_integer_oct_o
+  def test_integer_oct_o
     util_lex_token "0o52", :tINTEGER, 42
   end
 
-  def test_yylex_integer_oct_o_bad_range
-    util_bad_token "0o8"
+  def test_integer_oct_o_bad_range
+    util_bad_token "0o1283"
   end
 
-  def test_yylex_integer_oct_o_bad_underscores
+  def test_integer_oct_o_bad_underscores
     util_bad_token "0o1__23"
   end
 
-  def test_yylex_integer_oct_o_not_bad_none
+  def test_integer_oct_o_not_bad_none
     util_lex_token "0o ", :tINTEGER, 0
   end
 
-  def test_yylex_integer_trailing
+  def test_integer_trailing
     util_lex_token("1.to_s",
                    :tINTEGER, 1,
                    :tDOT, '.',
                    :tIDENTIFIER, 'to_s')
   end
 
-  def test_yylex_integer_underscore
+  def test_integer_underscore
     util_lex_token "4_2", :tINTEGER, 42
   end
 
-  def test_yylex_integer_underscore_bad
+  def test_integer_underscore_bad
     util_bad_token "4__2"
   end
 
-  def test_yylex_integer_zero
+  def test_integer_zero
     util_lex_token "0", :tINTEGER, 0
   end
 
-  def test_yylex_ivar
+  def test_ivar
     util_lex_token "@blah", :tIVAR, "@blah"
   end
 
-  def test_yylex_ivar_bad
+  def test_ivar_bad
     util_bad_token "@1"
   end
 
-  def test_yylex_ivar_bad_0_length
+  def test_ivar_bad_0_length
     util_bad_token "1+@\n", :tINTEGER, 1, :tPLUS, "+"
   end
 
-  def test_yylex_keyword_expr
+  def test_keyword_expr
     @lex.state = :expr_endarg
 
     util_lex_token("if", :kIF_MOD, "if")
@@ -1031,11 +1065,11 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_beg, @lex.state
   end
 
-  def test_yylex_lt
+  def test_lt
     util_lex_token "<", :tLT, "<"
   end
 
-  def test_yylex_lt2
+  def test_lt2
     util_lex_token("a <\< b",
                    :tIDENTIFIER, "a",
                    :tLSHFT, "<\<",
@@ -1043,81 +1077,75 @@ class TestLexer < MiniTest::Unit::TestCase
 
   end
 
-  def test_yylex_lt2_equals
+  def test_lt2_equals
     util_lex_token("a <\<= b",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, "<\<",
                    :tIDENTIFIER, "b")
   end
 
-  def test_yylex_lt_equals
+  def test_lt_equals
     util_lex_token "<=", :tLEQ, "<="
   end
 
-  def test_yylex_minus
+  def test_minus
     util_lex_token("1 - 2",
                    :tINTEGER, 1,
                    :tMINUS, "-",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_minus_equals
+  def test_minus_equals
     @lex.state = :expr_end
 
     util_lex_token "-=", :tOP_ASGN, "-"
   end
 
-  def test_yylex_minus_method
+  def test_minus_method
     @lex.state = :expr_fname
     util_lex_token "-", :tMINUS, "-"
   end
 
-  def test_yylex_minus_unary_method
+  def test_minus_unary_method
     @lex.state = :expr_fname
     util_lex_token "-@", :tUMINUS, "-@"
   end
 
-  def test_yylex_minus_unary_number
+  def test_minus_unary_number
     util_lex_token("-42",
                    :tUMINUS_NUM, "-",
                    :tINTEGER, 42)
   end
 
-  def test_yylex_nth_ref
-    util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
+  def test_nth_ref
+    util_lex_token('[$1, $2, $3]',
                    :tLBRACK,  "[",
                    :tNTH_REF, 1, :tCOMMA, ",",
                    :tNTH_REF, 2, :tCOMMA, ",",
-                   :tNTH_REF, 3, :tCOMMA, ",",
-                   :tNTH_REF, 4, :tCOMMA, ",",
-                   :tNTH_REF, 5, :tCOMMA, ",",
-                   :tNTH_REF, 6, :tCOMMA, ",",
-                   :tNTH_REF, 7, :tCOMMA, ",",
-                   :tNTH_REF, 8, :tCOMMA, ",",
-                   :tNTH_REF, 9,
+                   :tNTH_REF, 3,
                    :tRBRACK,  "]")
   end
 
-  def test_yylex_open_bracket
+  def test_open_bracket
     util_lex_token("(", :tLPAREN, "(")
   end
 
-  def test_yylex_open_bracket_cmdarg
+  def test_open_bracket_cmdarg
     util_lex_token("m (", :tIDENTIFIER, "m",
                           :tLPAREN_ARG, "(")
   end
 
-  def test_yylex_open_bracket_exprarg
+  def test_open_bracket_exprarg
     util_lex_token("m(", :tIDENTIFIER, "m",
                           :tLPAREN2, "(")
   end
 
-  def test_yylex_open_curly_bracket
+  def test_open_curly_bracket
     util_lex_token("{",
                    :tLBRACE, "{")
   end
 
-  def test_yylex_open_curly_bracket_arg
+  def test_open_curly_bracket_arg
     util_lex_token("m { 3 }",
                    :tIDENTIFIER, "m",
                    :tLCURLY, "{",
@@ -1125,7 +1153,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRCURLY, "}")
   end
 
-  def test_yylex_open_curly_bracket_block
+  def test_open_curly_bracket_block
     @lex.state = :expr_endarg # seen m(3)
 
     util_lex_token("{ 4 }",
@@ -1134,7 +1162,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRCURLY, "}")
   end
 
-  def test_yylex_open_square_bracket_arg
+  def test_open_square_bracket_arg
     util_lex_token("m [ 3 ]",
                    :tIDENTIFIER, "m",
                    :tLBRACK, "[",
@@ -1142,7 +1170,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRBRACK, "]")
   end
 
-  def test_yylex_open_square_bracket_ary
+  def test_open_square_bracket_ary
     util_lex_token("[1, 2, 3]",
                    :tLBRACK, "[",
                    :tINTEGER, 1,
@@ -1153,7 +1181,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRBRACK, "]")
   end
 
-  def test_yylex_open_square_bracket_meth
+  def test_open_square_bracket_meth
     util_lex_token("m[3]",
                    :tIDENTIFIER, "m",
                    :tLBRACK2, "[",
@@ -1161,60 +1189,60 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tRBRACK, "]")
   end
 
-  def test_yylex_or
+  def test_or
     util_lex_token "|", :tPIPE, "|"
   end
 
-  def test_yylex_or2
+  def test_or2
     util_lex_token "||", :tOROP, "||"
   end
 
-  def test_yylex_or2_equals
+  def test_or2_equals
     util_lex_token "||=", :tOP_ASGN, "||"
   end
 
-  def test_yylex_or_equals
+  def test_or_equals
     util_lex_token "|=", :tOP_ASGN, "|"
   end
 
-  def test_yylex_percent
+  def test_percent
     util_lex_token("a % 2",
                    :tIDENTIFIER, "a",
                    :tPERCENT, "%",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_percent_equals
+  def test_percent_equals
     util_lex_token("a %= 2",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, "%",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_plus
+  def test_plus
     util_lex_token("1 + 1", # TODO state?
                    :tINTEGER, 1,
                    :tPLUS, "+",
                    :tINTEGER, 1)
   end
 
-  def test_yylex_plus_equals
+  def test_plus_equals
     @lex.state = :expr_end
 
     util_lex_token "+=", :tOP_ASGN, "+"
   end
 
-  def test_yylex_plus_method
+  def test_plus_method
     @lex.state = :expr_fname
     util_lex_token "+", :tPLUS, "+"
   end
 
-  def test_yylex_plus_unary_method
+  def test_plus_unary_method
     @lex.state = :expr_fname
     util_lex_token "+@", :tUPLUS, "+@"
   end
 
-  def test_yylex_numbers
+  def test_numbers
     util_lex_token "0b10", :tINTEGER, 2
     util_lex_token "0B10", :tINTEGER, 2
 
@@ -1253,28 +1281,28 @@ class TestLexer < MiniTest::Unit::TestCase
     util_bad_token "1__1"
   end
 
-  def test_yylex_plus_unary_number
+  def test_plus_unary_number
     util_lex_token("+42",
                    :tINTEGER, 42)
   end
 
-  def test_yylex_question__18
+  def test_question__18
     setup_lexer 18
 
     util_lex_token "?*", :tINTEGER, 42
   end
 
-  def test_yylex_question__19
+  def test_question__19
     setup_lexer 19
 
     util_lex_token "?*", :tSTRING, "*"
   end
 
-  def test_yylex_question_bad_eos
+  def test_question_bad_eos
     util_bad_token "?"
   end
 
-  def test_yylex_question_bad_ws
+  def test_question_bad_ws
     util_lex_token "? ",  :tEH, "?"
     util_lex_token "?\n", :tEH, "?"
     util_lex_token "?\t", :tEH, "?"
@@ -1283,7 +1311,7 @@ class TestLexer < MiniTest::Unit::TestCase
     util_lex_token "?\f", :tEH, "?"
   end
 
-  def test_yylex_question_ws_backslashed__18
+  def test_question_ws_backslashed__18
     setup_lexer 18
 
     @lex.state = :expr_beg
@@ -1300,7 +1328,7 @@ class TestLexer < MiniTest::Unit::TestCase
     util_lex_token "?\\f", :tINTEGER, 12
   end
 
-  def test_yylex_question_ws_backslashed__19
+  def test_question_ws_backslashed__19
     setup_lexer 19
 
     @lex.state = :expr_beg
@@ -1317,15 +1345,15 @@ class TestLexer < MiniTest::Unit::TestCase
     util_lex_token "?\\f", :tSTRING, "\f"
   end
 
-  def test_yylex_rbracket
+  def test_rbracket
     util_lex_token "]", :tRBRACK, "]"
   end
 
-  def test_yylex_rcurly
+  def test_rcurly
     util_lex_token "}", :tRCURLY, "}"
   end
 
-  def test_yylex_regexp
+  def test_regexp
     util_lex_token("/regexp/",
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regexp",
@@ -1333,7 +1361,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_ambiguous
+  def test_regexp_ambiguous
     util_lex_token("method /regexp/",
                    :tIDENTIFIER,     "method",
                    :tREGEXP_BEG,     "/",
@@ -1342,13 +1370,14 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_bad
+  def test_regexp_bad
     util_bad_token("/.*/xyz",
-                   :tREGEXP_BEG, "/",
-                   :tSTRING_CONTENT, ".*")
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, ".*",
+                   :tSTRING_END,     "/")
   end
 
-  def test_yylex_regexp_escape_C
+  def test_regexp_escape_C
     util_lex_token('/regex\\C-x/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\C-x",
@@ -1356,7 +1385,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_C_M
+  def test_regexp_escape_C_M
     util_lex_token('/regex\\C-\\M-x/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\C-\\M-x",
@@ -1364,7 +1393,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_C_M_craaaazy
+  def test_regexp_escape_C_M_craaaazy
     util_lex_token("/regex\\C-\\\n\\M-x/",
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\C-\\M-x",
@@ -1372,27 +1401,27 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_C_bad_dash
+  def test_regexp_escape_C_bad_dash
     util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_C_bad_dash_eos
+  def test_regexp_escape_C_bad_dash_eos
     util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_C_bad_dash_eos2
+  def test_regexp_escape_C_bad_dash_eos2
     util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_C_bad_eos
+  def test_regexp_escape_C_bad_eos
     util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_C_bad_eos2
+  def test_regexp_escape_C_bad_eos2
     util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_M
+  def test_regexp_escape_M
     util_lex_token('/regex\\M-x/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\M-x",
@@ -1400,7 +1429,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_M_C
+  def test_regexp_escape_M_C
     util_lex_token('/regex\\M-\\C-x/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\M-\\C-x",
@@ -1408,23 +1437,23 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_M_bad_dash
+  def test_regexp_escape_M_bad_dash
     util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_M_bad_dash_eos
+  def test_regexp_escape_M_bad_dash_eos
     util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_M_bad_dash_eos2
+  def test_regexp_escape_M_bad_dash_eos2
     util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_M_bad_eos
+  def test_regexp_escape_M_bad_eos
     util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_backslash_slash
+  def test_regexp_escape_backslash_slash
     util_lex_token('/\\//',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, '\\/',
@@ -1432,43 +1461,43 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_backslash_terminator
+  def test_regexp_escape_backslash_terminator
     util_lex_token('%r%blah\\%blah%',
-                   :tREGEXP_BEG,     "%r",
+                   :tREGEXP_BEG,     "%r%",
                    :tSTRING_CONTENT, "blah\\%blah",
                    :tSTRING_END,     "%",
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_backslash_terminator_meta1
+  def test_regexp_escape_backslash_terminator_meta1
     util_lex_token('%r{blah\\}blah}',
-                   :tREGEXP_BEG,     "%r",
+                   :tREGEXP_BEG,     "%r{",
                    :tSTRING_CONTENT, "blah\\}blah",
                    :tSTRING_END,     "}",
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_backslash_terminator_meta2
+  def test_regexp_escape_backslash_terminator_meta2
     util_lex_token('%r/blah\\/blah/',
-                   :tREGEXP_BEG,     "%r",
+                   :tREGEXP_BEG,     "%r/",
                    :tSTRING_CONTENT, "blah\\/blah",
                    :tSTRING_END,     "/",
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_backslash_terminator_meta3
+  def test_regexp_escape_backslash_terminator_meta3
     util_lex_token('%r/blah\\%blah/',
-                   :tREGEXP_BEG,     "%r",
+                   :tREGEXP_BEG,     "%r/",
                    :tSTRING_CONTENT, "blah\\%blah",
                    :tSTRING_END,     "/",
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_bad_eos
+  def test_regexp_escape_bad_eos
     util_bad_token '/regex\\', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_bs
+  def test_regexp_escape_bs
     util_lex_token('/regex\\\\regex/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\\\regex",
@@ -1476,7 +1505,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_c
+  def test_regexp_escape_c
     util_lex_token('/regex\\cxxx/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\cxxx",
@@ -1484,7 +1513,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_c_backslash
+  def test_regexp_escape_c_backslash
     util_lex_token('/regex\\c\\n/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\c\\n",
@@ -1492,7 +1521,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_chars
+  def test_regexp_escape_chars
     util_lex_token('/re\\tge\\nxp/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "re\\tge\\nxp",
@@ -1500,7 +1529,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_double_backslash
+  def test_regexp_escape_double_backslash
     regexp = '/[\\/\\\\]$/'
     util_lex_token(regexp,
                    :tREGEXP_BEG,     "/",
@@ -1509,7 +1538,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_hex
+  def test_regexp_escape_hex
     util_lex_token('/regex\\x61xp/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\x61xp",
@@ -1517,11 +1546,11 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_hex_bad
+  def test_regexp_escape_hex_bad
     util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
   end
 
-  def test_yylex_regexp_escape_hex_one
+  def test_regexp_escape_hex_one
     util_lex_token('/^[\\xd\\xa]{2}/on',
                    :tREGEXP_BEG,     '/',
                    :tSTRING_CONTENT, '^[\\xd\\xa]{2}',
@@ -1529,7 +1558,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     'on')
   end
 
-  def test_yylex_regexp_escape_oct1
+  def test_regexp_escape_oct1
     util_lex_token('/regex\\0xp/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\0xp",
@@ -1537,7 +1566,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_oct2
+  def test_regexp_escape_oct2
     util_lex_token('/regex\\07xp/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\07xp",
@@ -1545,7 +1574,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_oct3
+  def test_regexp_escape_oct3
     util_lex_token('/regex\\10142/',
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regex\\10142",
@@ -1553,7 +1582,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_escape_return
+  def test_regexp_escape_return
     util_lex_token("/regex\\\nregex/",
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, "regexregex",
@@ -1561,7 +1590,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "")
   end
 
-  def test_yylex_regexp_nm
+  def test_regexp_nm
     util_lex_token("/.*/nm",
                    :tREGEXP_BEG,     "/",
                    :tSTRING_CONTENT, ".*",
@@ -1569,25 +1598,25 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tREGEXP_OPT,     "nm")
   end
 
-  def test_yylex_rparen
+  def test_rparen
     util_lex_token ")", :tRPAREN, ")"
   end
 
-  def test_yylex_rshft
+  def test_rshft
     util_lex_token("a >> 2",
                    :tIDENTIFIER, "a",
                    :tRSHFT, ">>",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_rshft_equals
+  def test_rshft_equals
     util_lex_token("a >>= 2",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, ">>",
                    :tINTEGER, 2)
   end
 
-  def test_yylex_star
+  def test_star
     util_lex_token("a * ",
                    :tIDENTIFIER, "a",
                    :tSTAR2, "*")
@@ -1595,7 +1624,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_beg, @lex.state
   end
 
-  def test_yylex_star2
+  def test_star2
     util_lex_token("a ** ",
                    :tIDENTIFIER, "a",
                    :tPOW, "**")
@@ -1603,7 +1632,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_beg, @lex.state
   end
 
-  def test_yylex_star2_equals
+  def test_star2_equals
     util_lex_token("a **= ",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, "**")
@@ -1611,7 +1640,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_beg, @lex.state
   end
 
-  def test_yylex_star_arg
+  def test_star_arg
     @lex.state = :expr_arg
 
     util_lex_token(" *a",
@@ -1621,7 +1650,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_arg, @lex.state
   end
 
-  def test_yylex_star_arg_beg
+  def test_star_arg_beg
     @lex.state = :expr_beg
 
     util_lex_token("*a",
@@ -1631,7 +1660,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_arg, @lex.state
   end
 
-  def test_yylex_star_arg_beg_fname
+  def test_star_arg_beg_fname
     @lex.state = :expr_fname
 
     util_lex_token("*a",
@@ -1641,7 +1670,7 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_arg, @lex.state
   end
 
-  def test_yylex_star_equals
+  def test_star_equals
     util_lex_token("a *= ",
                    :tIDENTIFIER, "a",
                    :tOP_ASGN, "*")
@@ -1649,97 +1678,97 @@ class TestLexer < MiniTest::Unit::TestCase
     assert_equal :expr_beg, @lex.state
   end
 
-  def test_yylex_string_bad_eos
+  def test_string_bad_eos
     util_bad_token('%',
                    :tSTRING_BEG,     '%')
   end
 
-  def test_yylex_string_bad_eos_quote
+  def test_string_bad_eos_quote
     util_bad_token('%{nest',
                    :tSTRING_BEG,     '%}')
   end
 
-  def test_yylex_string_double
+  def test_string_double
     util_lex_token('"string"',
                    :tSTRING, "string")
   end
 
-  def test_yylex_string_double_escape_C
+  def test_string_double_escape_C
     util_lex_token('"\\C-a"',
                    :tSTRING, "\001")
   end
 
-  def test_yylex_string_double_escape_C_backslash
+  def test_string_double_escape_C_backslash
     util_lex_token('"\\C-\\\\"',
                    :tSTRING, "\034")
   end
 
-  def test_yylex_string_double_escape_C_escape
+  def test_string_double_escape_C_escape
     util_lex_token('"\\C-\\M-a"',
                    :tSTRING, "\201")
   end
 
-  def test_yylex_string_double_escape_C_question
+  def test_string_double_escape_C_question
     util_lex_token('"\\C-?"',
                    :tSTRING, "\177")
   end
 
-  def test_yylex_string_double_escape_M
+  def test_string_double_escape_M
     util_lex_token('"\\M-a"',
                    :tSTRING, "\341")
   end
 
-  def test_yylex_string_double_escape_M_backslash
+  def test_string_double_escape_M_backslash
     util_lex_token('"\\M-\\\\"',
                    :tSTRING, "\334")
   end
 
-  def test_yylex_string_double_escape_M_escape
+  def test_string_double_escape_M_escape
     util_lex_token('"\\M-\\C-a"',
                    :tSTRING, "\201")
   end
 
-  def test_yylex_string_double_escape_bs1
+  def test_string_double_escape_bs1
     util_lex_token('"a\\a\\a"',
                    :tSTRING, "a\a\a")
   end
 
-  def test_yylex_string_double_escape_bs2
+  def test_string_double_escape_bs2
     util_lex_token('"a\\\\a"',
                    :tSTRING, "a\\a")
   end
 
-  def test_yylex_string_double_escape_c
+  def test_string_double_escape_c
     util_lex_token('"\\ca"',
                    :tSTRING, "\001")
   end
 
-  def test_yylex_string_double_escape_c_escape
+  def test_string_double_escape_c_escape
     util_lex_token('"\\c\\M-a"',
                    :tSTRING, "\201")
   end
 
-  def test_yylex_string_double_escape_c_question
+  def test_string_double_escape_c_question
     util_lex_token('"\\c?"',
                    :tSTRING, "\177")
   end
 
-  def test_yylex_string_double_escape_chars
+  def test_string_double_escape_chars
     util_lex_token('"s\\tri\\ng"',
                    :tSTRING, "s\tri\ng")
   end
 
-  def test_yylex_string_double_escape_hex
+  def test_string_double_escape_hex
     util_lex_token('"n = \\x61\\x62\\x63"',
                    :tSTRING, "n = abc")
   end
 
-  def test_yylex_string_double_escape_octal
+  def test_string_double_escape_octal
     util_lex_token('"n = \\101\\102\\103"',
                    :tSTRING, "n = ABC")
   end
 
-  def test_yylex_string_double_interp
+  def test_string_double_interp
     util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_CONTENT, "blah #x a ",
@@ -1756,12 +1785,12 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     "\"")
   end
 
-  def test_yylex_string_double_nested_curlies
+  def test_string_double_nested_curlies
     util_lex_token('%{nest{one{two}one}nest}',
                    :tSTRING, "nest{one{two}one}nest")
   end
 
-  def test_yylex_string_double_no_interp
+  def test_string_double_no_interp
     util_lex_token("\"# blah\"",                                # pound first
                    :tSTRING, "# blah")
 
@@ -1769,19 +1798,19 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING, "blah # blah")
   end
 
-  def test_yylex_string_escape_x_single
+  def test_string_escape_x_single
     util_lex_token('"\\x0"',
                    :tSTRING, "\000")
   end
 
-  def test_yylex_string_pct_Q
+  def test_string_pct_Q
     util_lex_token("%Q[s1 s2]",
                    :tSTRING, "s1 s2")
   end
 
-  def test_yylex_string_pct_W
+  def test_string_pct_W
     util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
-                   :tWORDS_BEG,      "%W",
+                   :tWORDS_BEG,      "%W[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s2",
@@ -1791,9 +1820,9 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     ']')
   end
 
-  def test_yylex_string_pct_W_bs_nl
+  def test_string_pct_W_bs_nl
     util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
-                   :tWORDS_BEG,      "%W",
+                   :tWORDS_BEG,      "%W[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "\ns2",
@@ -1801,28 +1830,28 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     ']')
   end
 
-  def test_yylex_string_pct_angle
+  def test_string_pct_angle
     util_lex_token("%<blah>",
                    :tSTRING, "blah")
   end
 
-  def test_yylex_string_pct_other
+  def test_string_pct_other
     util_lex_token("%%blah%",
                    :tSTRING, "blah")
   end
 
-  def test_yylex_string_pct_w
+  def test_string_pct_w
     util_bad_token("%w[s1 s2 ",
-                   :tQWORDS_BEG,     "%w",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s2",
                    :tSPACE,              nil)
   end
 
-  def test_yylex_string_pct_w_bs_nl
+  def test_string_pct_w_bs_nl
     util_lex_token("%w[s1 \\\ns2]",
-                   :tQWORDS_BEG,     "%w",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "\ns2",
@@ -1830,9 +1859,9 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     ']')
   end
 
-  def test_yylex_string_pct_w_bs_sp
+  def test_string_pct_w_bs_sp
     util_lex_token("%w[s\\ 1 s\\ 2]",
-                   :tQWORDS_BEG,     "%w",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s 1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s 2",
@@ -1840,9 +1869,9 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     ']')
   end
 
-  def test_yylex_string_pct_w_tab
+  def test_string_pct_w_tab
     util_lex_token("%w[abc\tdef]",
-                   :tQWORDS_BEG,      "%w",
+                   :tQWORDS_BEG,      "%w[",
                    :tSTRING_CONTENT, "abc",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "def",
@@ -1850,42 +1879,45 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     ']')
   end
 
-  def test_yylex_string_single
+  def test_string_single
     util_lex_token("'string'",
                    :tSTRING, "string")
   end
 
-  def test_yylex_string_single_escape_chars
+  def test_string_single_escape_chars
     util_lex_token("'s\\tri\\ng'",
                    :tSTRING, "s\\tri\\ng")
   end
 
-  def test_yylex_string_single_nl
+  def test_string_single_nl
     util_lex_token("'blah\\\nblah'",
-                   :tSTRING, "blah\\\nblah")
+                   :tSTRING_BEG,     "'",
+                   :tSTRING_CONTENT, "blah\\\n",
+                   :tSTRING_CONTENT, "blah",
+                   :tSTRING_END,     "'")
   end
 
-  def test_yylex_symbol
+  def test_symbol
     util_lex_token(":symbol",
                    :tSYMBOL, "symbol")
   end
 
-  def test_yylex_symbol_bad_zero
+  def test_symbol_bad_zero
     util_bad_token(":\"blah\0\"",
                    :tSYMBEG, ":")
   end
 
-  def test_yylex_symbol_double
+  def test_symbol_double
     util_lex_token(":\"symbol\"",
                    :tSYMBOL, "symbol")
   end
 
-  def test_yylex_symbol_single
+  def test_symbol_single
     util_lex_token(":'symbol'",
                    :tSYMBOL, "symbol")
   end
 
-  def test_yylex_ternary
+  def test_ternary
     util_lex_token("a ? b : c",
                    :tIDENTIFIER, "a",
                    :tEH,         "?",
@@ -1911,39 +1943,42 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tEH,         "?")
   end
 
-  def test_yylex_tilde
+  def test_tilde
     util_lex_token "~", :tTILDE, "~"
   end
 
-  def test_yylex_tilde_unary
+  def test_tilde_unary
     @lex.state = :expr_fname
     util_lex_token "~@", :tTILDE, "~@"
   end
 
-  def test_yylex_uminus
+  def test_uminus
     util_lex_token("-blah",
                    :tUMINUS, "-",
                    :tIDENTIFIER, "blah")
   end
 
-  def test_yylex_underscore
+  def test_underscore
     util_lex_token("_var", :tIDENTIFIER, "_var")
   end
 
-  def test_yylex_underscore_end
-    @lex.source = "__END__\n"
+  def test_underscore_end
+    source_buffer = Parser::Source::Buffer.new('(underscore_end)')
+    source_buffer.source = "__END__\n"
+
+    @lex.source_buffer = source_buffer
 
     tok, = @lex.advance
     refute tok
   end
 
-  def test_yylex_uplus
+  def test_uplus
     util_lex_token("+blah",
                    :tUPLUS, "+",
                    :tIDENTIFIER, "blah")
   end
 
-  def test_yylex_if_unless_mod
+  def test_if_unless_mod
     util_lex_token("return if true unless false",
                    :kRETURN,      "return",
                    :kIF_MOD,      "if",
@@ -1952,7 +1987,7 @@ class TestLexer < MiniTest::Unit::TestCase
                    :kFALSE,       "false")
   end
 
-  def test_yylex_if_stmt
+  def test_if_stmt
     util_lex_token("if true\n return end",
                    :kIF,          "if",
                    :kTRUE,        "true",
@@ -1961,9 +1996,9 @@ class TestLexer < MiniTest::Unit::TestCase
                    :kEND,         "end")
   end
 
-  def test_yylex_static_env
+  def test_static_env
     env = Parser::StaticEnvironment.new
-    env.declare :a
+    env.declare "a"
 
     @lex.static_env = env
 
@@ -1973,4 +2008,213 @@ class TestLexer < MiniTest::Unit::TestCase
                    :tINTEGER,    42,
                    :tRBRACK,     "]")
   end
+
+  #
+  # Tests for bugs.
+  #
+  # These tests should be moved from nursery and properly
+  # categorized when it's clear how to do that.
+  #
+
+  def test_bug_sclass_joined
+    util_lex_token("class<<self",
+                   :kCLASS, "class",
+                   :tLSHFT, "<<",
+                   :kSELF,  "self")
+  end
+
+  def test_bug_expr_beg_div
+    @lex.state = :expr_beg
+    util_lex_token("/=/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "=",
+                   :tSTRING_END,     "/",
+                   :tREGEXP_OPT,     "")
+
+    @lex.state = :expr_beg
+    util_lex_token("/ = /",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, " = ",
+                   :tSTRING_END,     "/",
+                   :tREGEXP_OPT,     "")
+  end
+
+  def test_bug_expr_beg_percent
+    @lex.state = :expr_beg
+    util_lex_token("%=foo=",
+                   :tSTRING, "foo")
+
+    @lex.state = :expr_beg
+    util_lex_token("% = ",
+                   :tSTRING, "=")
+  end
+
+  def test_bug_expr_beg_document
+    @lex.state = :expr_beg
+    util_lex_token(" \n=begin\n=end\nend",
+                   :kEND,        "end")
+
+  end
+
+  def test_bug_expr_beg_number
+    @lex.state = :expr_beg
+    util_lex_token("86400_000_000",
+                   :tINTEGER,    86400000000)
+  end
+
+  def test_bug_expr_beg_backspace_nl
+    @lex.state = :expr_beg
+    util_lex_token("\n/foo/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "foo",
+                   :tSTRING_END,     "/",
+                   :tREGEXP_OPT,     "")
+  end
+
+  def test_bug_expr_arg_percent
+    @lex.state = :expr_arg
+    util_lex_token("%[",
+                   :tPERCENT, "%",
+                   :tLBRACK,  "[")
+
+    @lex.state = :expr_arg
+    util_lex_token("%=1",
+                   :tOP_ASGN,    "%",
+                   :tINTEGER,    1)
+
+    @lex.state = :expr_arg
+    util_lex_token(" %[1]",
+                   :tSTRING,     "1")
+
+    @lex.state = :expr_arg
+    util_lex_token(" %=1=",
+                   :tOP_ASGN,    "%",
+                   :tINTEGER,    1,
+                   :tEQL,        "=")
+  end
+
+  def test_bug_expr_arg_lt_lt
+    @lex.state = :expr_arg
+    util_lex_token("<<EOS\nEOS",
+                   :tLSHFT,      "<<",
+                   :tCONSTANT,   "EOS",
+                   :tNL,         nil,
+                   :tCONSTANT,   "EOS")
+
+    @lex.state = :expr_arg
+    util_lex_token(" <<EOS\nEOS",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_END,     "EOS",
+                   :tNL,             nil)
+  end
+
+  def test_bug_expr_arg_slash
+    @lex.state = :expr_arg
+    util_lex_token("/1",
+                   :tDIVIDE,    "/",
+                   :tINTEGER,   1)
+
+    @lex.state = :expr_arg
+    util_lex_token("/ 1",
+                   :tDIVIDE,    "/",
+                   :tINTEGER,   1)
+
+    @lex.state = :expr_arg
+    util_lex_token(" /1/",
+                   :tREGEXP_BEG,     "/",
+                   :tSTRING_CONTENT, "1",
+                   :tSTRING_END,     "/",
+                   :tREGEXP_OPT,     "")
+
+    @lex.state = :expr_arg
+    util_lex_token(" / 1",
+                   :tDIVIDE,    "/",
+                   :tINTEGER,   1)
+  end
+
+  def test_bug_heredoc_continuation
+    @lex.state = :expr_arg
+    util_lex_token(" <<EOS\nEOS\nend",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_END,     "EOS",
+                   :tNL,             nil,
+                   :kEND,            "end")
+  end
+
+  def test_bug_eh_symbol_no_newline
+    util_lex_token("?\"\nfoo",
+                   :tINTEGER,     34,
+                   :tNL,          nil,
+                   :tIDENTIFIER,  "foo")
+  end
+
+  def test_bug_expr_arg_newline
+    @lex.state = :expr_arg
+    util_lex_token("\nfoo",
+                   :tNL,          nil,
+                   :tIDENTIFIER,  "foo")
+
+    @lex.state = :expr_arg
+    util_lex_token(" \nfoo",
+                   :tNL,          nil,
+                   :tIDENTIFIER,  "foo")
+
+    @lex.state = :expr_arg
+    util_lex_token("#foo\nfoo",
+                   :tNL,          nil,
+                   :tIDENTIFIER,  "foo")
+  end
+
+  def test_bug_heredoc_backspace_nl
+    util_lex_token(" <<'XXX'\nf \\\nXXX\n",
+                   :tSTRING_BEG,     "'",
+                   :tSTRING_CONTENT, "f \\\n",
+                   :tSTRING_END,     "XXX",
+                   :tNL,             nil)
+  end
+
+  def test_bug_expr_dot_comment
+    util_lex_token("foo. #bar\nbaz",
+                   :tIDENTIFIER, 'foo',
+                   :tDOT,        '.',
+                   :tIDENTIFIER, 'baz')
+  end
+
+  def test_bug_expr_mid_comment
+    util_lex_token("rescue #bar\nprint",
+                   :kRESCUE,     'rescue',
+                   :tNL,         nil,
+                   :tIDENTIFIER, 'print')
+  end
+
+  def test_bug_expr_end_colon
+    util_lex_token("'foo':'bar'",
+                   :tSTRING, 'foo',
+                   :tCOLON,  ':',
+                   :tSTRING, 'bar')
+  end
+
+  def test_bug_symbol_newline
+    util_lex_token(":foo\n",
+                   :tSYMBOL, 'foo',
+                   :tNL,     nil)
+
+    util_lex_token(":foo=\n",
+                   :tSYMBOL, 'foo=',
+                   :tNL,     nil)
+  end
+
+  def test_bug_ragel_stack
+    util_lex_token("\"\#{$2 ? $2 : 1}\"",
+                   :tSTRING_BEG,      "\"",
+                   :tSTRING_DBEG,     "\#{",
+                   :tNTH_REF,         2,
+                   :tEH,              "?",
+                   :tNTH_REF,         2,
+                   :tCOLON,           ":",
+                   :tINTEGER,         1,
+                   :tRCURLY,          "}",
+                   :tSTRING_END,      "\"")
+  end
+
 end