syntax 0.5.0 → 0.7.0

data/test/syntax/tc_ruby.rb

@@ -1,518 +1,666 @@
-$:.unshift "../../lib"
+$:.unshift File.dirname(__FILE__) +"/../../lib"
 
 require 'test/unit'
-require 'syntax/ruby'
+require 'syntax/lang/ruby'
 
 class TC_Syntax_Ruby < Test::Unit::TestCase
 
+  def tokenize( string )
+    @tokens = []
+    @ruby.tokenize( string ) { |tok| @tokens << tok }
+  end
+
+  def assert_next_token(group, lexeme, instruction=:none)
+    assert false, "no tokens in stack" if @tokens.nil? or @tokens.empty?
+    assert_equal [group, lexeme, instruction],
+      [@tokens.first.group, @tokens.first, @tokens.shift.instruction]
+  end
+
+  def assert_no_next_token
+    assert @tokens.empty?
+  end
+
+  def skip_token( n=1 )
+    n.times { @tokens.shift } unless @tokens.nil? || @tokens.empty?
+  end
+
   def setup
     @ruby = Syntax::Ruby.new
   end
 
   def test_empty
-    called = false
-    @ruby.tokenize( "" ) { |tok| called = true }
-    assert !called
+    tokenize( "" )
+    assert_no_next_token
   end
 
   def test_constant
-    called = false
-    @ruby.tokenize( "Foo" ) do |tok|
-      called = true
-      assert_equal :constant, tok.group
-      assert_equal "Foo", tok
-    end
-    assert called
+    tokenize( "Foo" )
+    assert_next_token :constant, "Foo"
   end
 
   def test_ident
-    called = false
-    @ruby.tokenize( "foo" ) do |tok|
-      called = true
-      assert_equal :ident, tok.group
-      assert_equal "foo", tok
-    end
-    assert called
+    tokenize( "foo" )
+    assert_next_token :ident, "foo"
   end
 
   def test_comment_eol
-    called = false
-    @ruby.tokenize( "# a comment" ) do |tok|
-      called = true
-      assert_equal :comment, tok.group
-      assert_equal "# a comment", tok
-    end
-    assert called
+    tokenize( "# a comment\nfoo" )
+    assert_next_token :comment, "# a comment"
+    assert_next_token :normal, "\n"
+    assert_next_token :ident, "foo"
   end
 
   def test_comment_block
-    called = false
-    @ruby.tokenize( "=begin\nthis is a comment\n=end" ) do |tok|
-      called = true
-      assert_equal :comment, tok.group
-      assert_equal "=begin\nthis is a comment\n=end", tok
-    end
-    assert called
+    tokenize( "=begin\nthis is a comment\n=end\nnoncomment" )
+    assert_next_token :comment, "=begin\nthis is a comment\n=end"
+    assert_next_token :normal, "\n"
+    assert_next_token :ident, "noncomment"
   end
 
   def test_keyword
     Syntax::Ruby::KEYWORDS.each do |word|
-      tok = []
-      @ruby.tokenize( word ) { |t| tok << t }
-      assert_equal [ :keyword, word ], [ tok.first.group, tok.first ]
+      tokenize( word )
+      assert_next_token :keyword, word
     end
     Syntax::Ruby::KEYWORDS.each do |word|
-      tok = []
-      @ruby.tokenize( "foo.#{word}" ) { |t| tok << t }
-      tok.shift
-      tok.shift
-      assert_equal [ :ident, word ], [ tok.first.group, tok.first ]
+      tokenize( "foo.#{word}" )
+      skip_token 2
+      assert_next_token :ident, word
     end
   end
 
   def test__END__
-    called = false
-    @ruby.tokenize( "__END__\n\nblah blah blah" ) do |tok|
-      called = true
-      assert_equal :comment, tok.group
-      assert_equal "__END__\n\nblah blah blah", tok
-    end
-    assert called
+    tokenize( "__END__\n\nblah blah blah" )
+    assert_next_token :comment, "__END__\n\nblah blah blah"
   end
 
   def test_def_paren
-    tok = nil
-    @ruby.start( "def  foo(bar)" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "def  ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "foo", tok
-    assert_equal :method, tok.group
+    tokenize( "def  foo(bar)" )
+    assert_next_token :keyword, "def  "
+    assert_next_token :method, "foo"
+    assert_next_token :punct, "("
+    assert_next_token :ident, "bar"
+    assert_next_token :punct, ")"
   end
 
   def test_def_space
-    tok = nil
-    @ruby.start( "def  foo bar" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "def  ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "foo", tok
-    assert_equal :method, tok.group
+    tokenize( "def  foo bar" )
+    assert_next_token :keyword, "def  "
+    assert_next_token :method, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :ident, "bar"
   end
 
   def test_def_semicolon
-    tok = nil
-    @ruby.start( "def  foo;" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "def  ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "foo", tok
-    assert_equal :method, tok.group
+    tokenize( "def  foo;" )
+    assert_next_token :keyword, "def  "
+    assert_next_token :method, "foo"
+    assert_next_token :punct, ";"
   end
 
   def test_class_space
-    tok = nil
-    @ruby.start( "class    Foo\n" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "class    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo", tok
-    assert_equal :class, tok.group
+    tokenize( "class    Foo\n" )
+    assert_next_token :keyword, "class    "
+    assert_next_token :class, "Foo"
+    assert_next_token :normal, "\n"
   end
 
   def test_class_semicolon
-    tok = nil
-    @ruby.start( "class    Foo;" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "class    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo", tok
-    assert_equal :class, tok.group
+    tokenize( "class    Foo;" )
+    assert_next_token :keyword, "class    "
+    assert_next_token :class, "Foo"
+    assert_next_token :punct, ";"
   end
 
   def test_class_extend
-    tok = nil
-    @ruby.start( "class    Foo< Bang" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "class    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo", tok
-    assert_equal :class, tok.group
+    tokenize( "class    Foo< Bang" )
+    assert_next_token :keyword, "class    "
+    assert_next_token :class, "Foo"
+    assert_next_token :punct, "<"
+    assert_next_token :normal, " "
+    assert_next_token :constant, "Bang"
   end
 
   def test_module_space
-    tok = nil
-    @ruby.start( "module    Foo\n" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "module    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo", tok
-    assert_equal :module, tok.group
+    tokenize( "module    Foo\n" )
+    assert_next_token :keyword, "module    "
+    assert_next_token :module, "Foo"
+    assert_next_token :normal, "\n"
   end
 
   def test_module_semicolon
-    tok = nil
-    @ruby.start( "module    Foo;" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "module    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo", tok
-    assert_equal :module, tok.group
+    tokenize( "module    Foo;" )
+    assert_next_token :keyword, "module    "
+    assert_next_token :module, "Foo"
+    assert_next_token :punct, ";"
   end
 
   def test_module_other
-    tok = nil
-    @ruby.start( "module    Foo!\n" ) { |t| tok = t }
-    @ruby.step
-    assert_equal "module    ", tok
-    assert_equal :keyword, tok.group
-    @ruby.step
-    assert_equal "Foo!", tok
-    assert_equal :module, tok.group
+    tokenize( "module    Foo!\n" )
+    assert_next_token :keyword, "module    "
+    assert_next_token :module, "Foo!"
   end
 
   def test_scope_operator
-    tok = []
-    @ruby.tokenize( "Foo::Bar" ) { |t| tok << t }
-    assert_equal "Foo", tok.shift
-    assert_equal "::", tok.first
-    assert_equal :punct, tok.shift.group
-    assert_equal "Bar", tok.first
+    tokenize( "Foo::Bar" )
+    assert_next_token :constant, "Foo"
+    assert_next_token :punct, "::"
+    assert_next_token :constant, "Bar"
   end
 
   def test_symbol_dquote
-    tok = []
-    @ruby.tokenize( ':"foo"' ) { |t| tok << t }
-    assert_equal ':"foo"', tok.first
-    assert_equal :symbol, tok.first.group
+    tokenize( ':"foo"' )
+    assert_next_token :symbol, ':"'
+    assert_next_token :symbol, '', :region_open
+    assert_next_token :symbol, 'foo'
+    assert_next_token :symbol, '', :region_close
+    assert_next_token :symbol, '"'
+    assert_no_next_token
   end
 
   def test_symbol_squote
-    tok = []
-    @ruby.tokenize( ":'foo'" ) { |t| tok << t }
-    assert_equal ":'foo'", tok.first
-    assert_equal :symbol, tok.first.group
+    tokenize( ":'foo'" )
+    assert_next_token :symbol, ":'"
+    assert_next_token :symbol, "", :region_open
+    assert_next_token :symbol, "foo"
+    assert_next_token :symbol, "", :region_close
+    assert_next_token :symbol, "'"
+    assert_no_next_token
   end
 
   def test_symbol
-    tok = []
-    @ruby.tokenize( ":foo_bar?" ) { |t| tok << t }
-    assert_equal ":foo_bar?", tok.first
-    assert_equal :symbol, tok.first.group
+    tokenize( ":foo_bar?" )
+    assert_next_token :symbol, ":foo_bar?"
   end
 
   def test_char
-    tok = []
-    @ruby.tokenize( "?." ) { |t| tok << t }
-    assert_equal "?.", tok.first
-    assert_equal :char, tok.first.group
+    tokenize( "?." )
+    assert_next_token :char, "?."
 
-    tok.clear
-    @ruby.tokenize( '?\n' ) { |t| tok << t }
-    assert_equal '?\n', tok.first
-    assert_equal :char, tok.first.group
+    tokenize( '?\n' )
+    assert_next_token :char, '?\n'
   end
 
   def test_specials
     %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tok = []
-      @ruby.tokenize( word ) { |t| tok << t }
-      assert_equal word, tok.first
-      assert_equal :constant, tok.first.group
+      tokenize( word )
+      assert_next_token :constant, word
     end
 
     %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tok = []
-      @ruby.tokenize( "#{word}?" ) { |t| tok << t }
-      assert_equal "#{word}?", tok.first
-      assert_equal :ident, tok.first.group
+      tokenize( "#{word}?" )
+      assert_next_token :ident, "#{word}?"
     end
 
     %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tok = []
-      @ruby.tokenize( "#{word}!" ) { |t| tok << t }
-      assert_equal "#{word}!", tok.first
-      assert_equal :ident, tok.first.group
+      tokenize( "#{word}!" )
+      assert_next_token :ident, "#{word}!"
     end
 
     %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tok = []
-      @ruby.tokenize( "x.#{word}" ) { |t| tok << t }
-      tok.shift
-      tok.shift
-      assert_equal word, tok.first
-      assert_equal :ident, tok.first.group
+      tokenize( "x.#{word}" )
+      skip_token 2
+      assert_next_token :ident, word
     end
   end
 
   def test_pct_r
-    tok = []
-    @ruby.tokenize( '%r{foo#{x}bar}' ) { |t| tok << t }
-    assert_equal [ :punct, "%r{" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "}" ], [ tok.first.group, tok.shift ]
-
-    tok = []
-    @ruby.tokenize( '%r-foo#{x}bar-' ) { |t| tok << t }
-    assert_equal [ :punct, "%r-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%r{foo#{x}bar}' )
+    assert_next_token :punct, "%r{"
+    assert_next_token :regex, "", :region_open
+    assert_next_token :regex, "foo"
+    assert_next_token :expr, '#{x}'
+    assert_next_token :regex, "bar"
+    assert_next_token :regex, "", :region_close
+    assert_next_token :punct, "}"
+
+    tokenize( '%r-foo#{x}bar-' )
+    assert_next_token :punct, "%r-"
+    assert_next_token :regex, "", :region_open
+    assert_next_token :regex, "foo"
+    assert_next_token :expr, '#{x}'
+    assert_next_token :regex, "bar"
+    assert_next_token :regex, "", :region_close
+    assert_next_token :punct, "-"
+  end
+
+  def test_pct_r_with_wakas
+    tokenize '%r<foo#{x}bar> foo'
+    assert_next_token :punct, "%r<"
+    assert_next_token :regex, "", :region_open
+    assert_next_token :regex, "foo"
+    assert_next_token :expr, '#{x}'
+    assert_next_token :regex, "bar"
+    assert_next_token :regex, "", :region_close
+    assert_next_token :punct, ">"
+    assert_next_token :normal, " "
+    assert_next_token :ident, "foo"
+  end
+
+  def test_pct_w_brace
+    tokenize( '%w{foo bar baz}' )
+    assert_next_token :punct, "%w{"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'foo bar baz'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "}"
   end
 
   def test_pct_w
-    tok = []
-    @ruby.tokenize( '%w-foo#{x} bar baz-' ) { |t| tok << t }
-    assert_equal [ :punct, "%w-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'foo#{x} bar baz' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%w-foo#{x} bar baz-' )
+    assert_next_token :punct, "%w-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'foo#{x} bar baz'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_pct_q
-    tok = []
-    @ruby.tokenize( '%q-hello #{world}-' ) { |t| tok << t }
-    assert_equal [ :punct, "%q-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'hello #{world}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%q-hello #{world}-' )
+    assert_next_token :punct, "%q-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'hello #{world}'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
  
   def test_pct_s
-    tok = []
-    @ruby.tokenize( '%s-hello #{world}-' ) { |t| tok << t }
-    assert_equal [ :punct, "%s-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :symbol, 'hello #{world}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%s-hello #{world}-' )
+    assert_next_token :punct, "%s-"
+    assert_next_token :symbol, '', :region_open
+    assert_next_token :symbol, 'hello #{world}'
+    assert_next_token :symbol, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_pct_W
-    tok = []
-    @ruby.tokenize( '%W-foo#{x} bar baz-' ) { |t| tok << t }
-    assert_equal [ :punct, "%W-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'foo' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, ' bar baz' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%W-foo#{x} bar baz-' )
+    assert_next_token :punct, "%W-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'foo'
+    assert_next_token :expr, '#{x}'
+    assert_next_token :string, ' bar baz'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_pct_Q
-    tok = []
-    @ruby.tokenize( '%Q-hello #{world}-' ) { |t| tok << t }
-    assert_equal [ :punct, "%Q-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%Q-hello #{world}-' )
+    assert_next_token :punct, "%Q-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'hello '
+    assert_next_token :expr, '#{world}'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_pct_x
-    tok = []
-    @ruby.tokenize( '%x-ls /blah/#{foo}-' ) { |t| tok << t }
-    assert_equal [ :punct, "%x-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'ls /blah/' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{foo}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%x-ls /blah/#{foo}-' )
+    assert_next_token :punct, "%x-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'ls /blah/'
+    assert_next_token :expr, '#{foo}'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_pct_string
-    tok = []
-    @ruby.tokenize( '%-hello #{world}-' ) { |t| tok << t }
-    assert_equal [ :punct, "%-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+    tokenize( '%-hello #{world}-' )
+    assert_next_token :punct, "%-"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'hello '
+    assert_next_token :expr, '#{world}'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, "-"
   end
 
   def test_bad_pct_string
-    tok = []
-    @ruby.tokenize( '%0hello #{world}0' ) { |t| tok << t }
-    assert_equal [ :punct, "%" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :number, '0' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :ident, 'hello' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, ' ' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :comment, '#{world}0' ], [ tok.first.group, tok.shift ]
+    tokenize( '%0hello #{world}0' )
+    assert_next_token :punct, "%"
+    assert_next_token :number, '0'
+    assert_next_token :ident, 'hello'
+    assert_next_token :normal, ' '
+    assert_next_token :comment, '#{world}0'
   end
 
   def test_shift_left
-    tok = []
-    @ruby.tokenize( 'foo << 5' ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :number, "5" ], [ tok.first.group, tok.shift ]
+    tokenize( 'foo << 5' )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<"
+    assert_next_token :normal, " "
+    assert_next_token :number, "5"
+  end
+
+  def test_shift_left_no_white
+    tokenize( 'foo<<5' )
+    assert_next_token :ident, "foo"
+    assert_next_token :punct, "<<"
+    assert_next_token :number, "5"
   end
 
   def test_here_doc_no_opts
-    tok = []
-    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\nEOF" ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    tokenize( "foo <<EOF\n  foo\n  bar\n  baz\nEOF" )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<"
+    assert_next_token :constant, "EOF"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  foo\n  bar\n  baz\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "EOF"
   end
 
   def test_here_doc_no_opts_missing_end
-    tok = []
-    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\n EOF" ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  foo\n  bar\n  baz\n EOF" ], [ tok.first.group, tok.shift ]
+    tokenize( "foo <<EOF\n  foo\n  bar\n  baz\n EOF" )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<"
+    assert_next_token :constant, "EOF"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  foo\n  bar\n  baz\n EOF"
+    assert_no_next_token
   end
 
   def test_here_doc_float_right
-    tok = []
-    @ruby.tokenize( "foo <<-EOF\n  foo\n  bar\n  baz\n  EOF" ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<-" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "  EOF" ], [ tok.first.group, tok.shift ]
+    tokenize( "foo <<-EOF\n  foo\n  bar\n  baz\n  EOF" )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<-"
+    assert_next_token :constant, "EOF"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  foo\n  bar\n  baz\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "  EOF"
   end
 
   def test_here_doc_single_quotes
-    tok = []
-    @ruby.tokenize( "foo <<'EOF'\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  foo\#{x}\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    tokenize( "foo <<'EOF'\n  foo\#{x}\n  bar\n  baz\nEOF" )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<'"
+    assert_next_token :constant, "EOF"
+    assert_next_token :punct, "'"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  foo\#{x}\n  bar\n  baz\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "EOF"
   end
 
   def test_here_doc_double_quotes
-    tok = []
-    @ruby.tokenize( "foo <<\"EOF\"\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
-    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "<<\"" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "\"" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  foo" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, "\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    tokenize( "foo <<\"EOF\"\n  foo\#{x}\n  bar\n  baz\nEOF" )
+    assert_next_token :ident, "foo"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<\""
+    assert_next_token :constant, "EOF"
+    assert_next_token :punct, "\""
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  foo"
+    assert_next_token :expr, '#{x}'
+    assert_next_token :string, "\n  bar\n  baz\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "EOF"
   end
 
   def test_space
-    tok = []
-    @ruby.tokenize( "\n  \t\t\n\n\r\n" ) { |t| tok << t }
-    assert_equal [ :normal, "\n  \t\t\n\n\r\n" ], [ tok.first.group, tok.shift ]
+    tokenize( "\n  \t\t\n\n\r\n" )
+    assert_next_token :normal, "\n  \t\t\n\n\r\n"
   end
 
   def test_number
-    tok = []
-    @ruby.tokenize( "1 1.0 1e5 1.0e5 1_2.5 1_2.5_2 1_2.5_2e3_2" ) { |t| tok << t }
-    assert_equal [ :number, "1" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1.0" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1e5" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1.0e5" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1_2.5" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1_2.5_2" ], [ tok.first.group, tok.shift ]
-    tok.shift
-    assert_equal [ :number, "1_2.5_2e3_2" ], [ tok.first.group, tok.shift ]
+    tokenize( "1 1.0 1e5 1.0e5 1_2.5 1_2.5_2 1_2.5_2e3_2" )
+    assert_next_token :number, "1"
+    skip_token
+    assert_next_token :number, "1.0"
+    skip_token
+    assert_next_token :number, "1e5"
+    skip_token
+    assert_next_token :number, "1.0e5"
+    skip_token
+    assert_next_token :number, "1_2.5"
+    skip_token
+    assert_next_token :number, "1_2.5_2"
+    skip_token
+    assert_next_token :number, "1_2.5_2e3_2"
   end
 
   def test_dquoted_string
-    tok = []
-    @ruby.tokenize( '"foo #{x} bar\"\n\tbaz\xA5b\5\1234"' ) { |t| tok << t }
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'foo ' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, ' bar' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\"\n\t' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'baz' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\xA5' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'b' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\5\123' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, '4' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+    tokenize( '"foo #{x} bar\"\n\tbaz\xA5b\5\1234"' )
+    assert_next_token :punct, '"'
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'foo '
+    assert_next_token :expr, '#{x}'
+    assert_next_token :string, ' bar'
+    assert_next_token :escape, '\"\n\t'
+    assert_next_token :string, 'baz'
+    assert_next_token :escape, '\xA5'
+    assert_next_token :string, 'b'
+    assert_next_token :escape, '\5\123'
+    assert_next_token :string, '4'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct, '"'
   end
 
   def test_squoted_string
-    tok = []
-    @ruby.tokenize( '\'foo #{x} bar\\\'\n\tbaz\\\\\xA5b\5\1234\'' ) { |t| tok << t }
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, 'foo #{x} bar' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\\\'' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, '\n\tbaz' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :expr, '\\\\' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :string, '\xA5b\5\1234' ], [ tok.first.group, tok.shift ]
-    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    tokenize( '\'foo #{x} bar\\\'\n\tbaz\\\\\xA5b\5\1234\'' )
+    assert_next_token :punct, "'"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, 'foo #{x} bar'
+    assert_next_token :escape, '\\\''
+    assert_next_token :string, '\n\tbaz'
+    assert_next_token :escape, '\\\\'
+    assert_next_token :string, '\xA5b\5\1234'
+    assert_next_token :string, "", :region_close
+    assert_next_token :punct, "'"
   end
 
   def test_dot_selector
-    tok = []
-    @ruby.tokenize( 'foo.nil' ) { |t| tok << t }
-    tok.shift
-    assert_equal [ :punct, "." ], [ tok.first.group, tok.shift ]
-    assert_equal [ :ident, "nil" ], [ tok.first.group, tok.shift ]
+    tokenize( 'foo.nil' )
+    skip_token
+    assert_next_token :punct, "."
+    assert_next_token :ident, "nil"
   end
 
   def test_dot_range_inclusive
-    tok = []
-    @ruby.tokenize( 'foo..nil' ) { |t| tok << t }
-    tok.shift
-    assert_equal [ :punct, ".." ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+    tokenize( 'foo..nil' )
+    skip_token
+    assert_next_token :punct, ".."
+    assert_next_token :constant, "nil"
   end
 
   def test_dot_range_exclusive
-    tok = []
-    @ruby.tokenize( 'foo...nil' ) { |t| tok << t }
-    tok.shift
-    assert_equal [ :punct, "..." ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+    tokenize( 'foo...nil' )
+    skip_token
+    assert_next_token :punct, "..."
+    assert_next_token :constant, "nil"
   end
 
   def test_dot_range_many
-    tok = []
-    @ruby.tokenize( 'foo.....nil' ) { |t| tok << t }
-    tok.shift
-    assert_equal [ :punct, "....." ], [ tok.first.group, tok.shift ]
-    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+    tokenize( 'foo.....nil' )
+    skip_token
+    assert_next_token :punct, "....."
+    assert_next_token :constant, "nil"
   end
 
   def test_attribute
-    tok = []
-    @ruby.tokenize( '@var_foo' ) { |t| tok << t }
-    assert_equal [ :attribute, "@var_foo" ], [ tok.first.group, tok.shift ]
+    tokenize( '@var_foo' )
+    assert_next_token :attribute, "@var_foo"
   end
 
   def test_global
-    tok = []
-    @ruby.tokenize( '$var_foo' ) { |t| tok << t }
-    assert_equal [ :global, "$var_foo" ], [ tok.first.group, tok.shift ]
-    tok = []
-    @ruby.tokenize( '$12' ) { |t| tok << t }
-    assert_equal [ :global, "$12" ], [ tok.first.group, tok.shift ]
-    tok = []
-    @ruby.tokenize( '$/f' ) { |t| tok << t }
-    assert_equal [ :global, "$/" ], [ tok.first.group, tok.shift ]
-    tok = []
-    @ruby.tokenize( "$\n" ) { |t| tok << t }
-    assert_equal [ :global, "$" ], [ tok.first.group, tok.shift ]
+    tokenize( '$var_foo' )
+    assert_next_token :global, "$var_foo"
+    tokenize( '$12' )
+    assert_next_token :global, "$12"
+    tokenize( '$/f' )
+    assert_next_token :global, "$/"
+    tokenize( "$\n" )
+    assert_next_token :global, "$"
+  end
+
+  def test_paren_delimiter
+    tokenize( '%w(a)' )
+    assert_next_token :punct, "%w("
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "a"
+    assert_next_token :string, "", :region_close
+    assert_next_token :punct, ")"
+  end
+
+  def test_division
+    tokenize( 'm / 3' )
+    assert_next_token :ident, "m"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "/"
+    assert_next_token :normal, " "
+    assert_next_token :number, "3"
+  end
+
+  def test_regex
+    tokenize( 'm =~ /3/' )
+    assert_next_token :ident, "m"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "=~"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "/"
+    assert_next_token :regex, "", :region_open
+    assert_next_token :regex, "3"
+    assert_next_token :regex, "", :region_close
+    assert_next_token :punct, "/"
+  end
+
+  def test_heredoc_with_trailing_text
+    tokenize( "foo('here', <<EOF)\n  A heredoc.\nEOF\nfoo" )
+    assert_next_token :ident,  "foo"
+    assert_next_token :punct,  "('"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'here'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct,  "',"
+    assert_next_token :normal, ' '
+    assert_next_token :punct,  '<<'
+    assert_next_token :constant, "EOF"
+    assert_next_token :punct,  ')'
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n  A heredoc.\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "EOF"
+    assert_next_token :normal, "\n"
+    assert_next_token :ident,  "foo"
+  end
+
+  def test_multiple_heredocs
+    tokenize( <<'TEST' )
+foo('here', <<EOF, 'there', <<-'FOO', 'blah')
+First heredoc, right here.
+Expressions are #{allowed}
+EOF
+    Another heredoc, immediately after the first.
+    Expressions are not #{allowed}
+  FOO
+TEST
+    assert_next_token :ident,  "foo"
+    assert_next_token :punct,  "('"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'here'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct,  "',"
+    assert_next_token :normal, ' '
+    assert_next_token :punct,  '<<'
+    assert_next_token :constant, "EOF"
+    assert_next_token :punct,  ','
+    assert_next_token :normal, ' '
+    assert_next_token :punct,  "'"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'there'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct,  "',"
+    assert_next_token :normal, ' '
+    assert_next_token :punct,  "<<-'"
+    assert_next_token :constant, "FOO"
+    assert_next_token :punct,  "',"
+    assert_next_token :normal, ' '
+    assert_next_token :punct,  "'"
+    assert_next_token :string, '', :region_open
+    assert_next_token :string, 'blah'
+    assert_next_token :string, '', :region_close
+    assert_next_token :punct,  "')"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\nFirst heredoc, right here.\nExpressions are "
+    assert_next_token :expr, '#{allowed}'
+    assert_next_token :string, "\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "EOF"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\n    Another heredoc, immediately after the first.\n    Expressions are not \#{allowed}\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "  FOO"
+  end
+
+  def test_carldr_bad_heredoc_001
+    tokenize( <<'TEST' )
+str = <<END
+here document #{1 + 1}
+END
+
+if str
+TEST
+
+    assert_next_token :ident, "str"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "="
+    assert_next_token :normal, " "
+    assert_next_token :punct, "<<"
+    assert_next_token :constant, "END"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\nhere document "
+    assert_next_token :expr, '#{1 + 1}'
+    assert_next_token :string, "\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "END"
+    assert_next_token :normal, "\n\n"
+    assert_next_token :keyword, "if"
+    assert_next_token :normal, " "
+    assert_next_token :ident, "str"
+  end
+
+  def test_regex_after_keyword
+    tokenize "when /[0-7]/\nfoo"
+    assert_next_token :keyword, "when"
+    assert_next_token :normal, " "
+    assert_next_token :punct, "/"
+    assert_next_token :regex, "", :region_open
+    assert_next_token :regex, "[0-7]"
+    assert_next_token :regex, "", :region_close
+    assert_next_token :punct, "/"
+    assert_next_token :normal, "\n"
+    assert_next_token :ident, "foo"
+  end
+
+  def test_heredoc_after_lparen
+    tokenize "foo(<<SRC, obj)\nblah blah\nSRC\nfoo"
+    assert_next_token :ident, "foo"
+    assert_next_token :punct, "(<<"
+    assert_next_token :constant, "SRC"
+    assert_next_token :punct, ","
+    assert_next_token :normal, " "
+    assert_next_token :ident, "obj"
+    assert_next_token :punct, ")"
+    assert_next_token :string, "", :region_open
+    assert_next_token :string, "\nblah blah\n"
+    assert_next_token :string, "", :region_close
+    assert_next_token :constant, "SRC"
+    assert_next_token :normal, "\n"
+    assert_next_token :ident, "foo"
+  end
+
+  def test_division_after_parens
+    tokenize "(a+b)/2"
+    assert_next_token :punct, "("
+    assert_next_token :ident, "a"
+    assert_next_token :punct, "+"
+    assert_next_token :ident, "b"
+    assert_next_token :punct, ")/"
+    assert_next_token :number, "2"
   end
-
 end