syntax 1.1.0 → 1.2.0

data/CHANGELOG

@@ -1,6 +1,9 @@
 Syntax: a syntax highlighting library for Ruby.
 
-1.1.0 In Progress
+1.2.0 02 Jan 2014
+  Cleaned up Gemspec, added license and homepage - @grosser.
+
+1.1.0 11 Dec 2013
   Published from https://github.com/dblock/syntax, a now maintained fork.
   The project builds again and runs, fixes by @dblock, @distler.
 

data/README.rdoc

@@ -39,11 +39,3 @@ Tokenizing is straightforward process. Each time a new token is discovered by th
 * <tt>token.instruction</tt> is an instruction used to determine how this token should be treated. It will be <tt>:none</tt> for normal tokens, <tt>:region_open</tt> if the token starts a nested region, and <tt>:region_close</tt> if it closes the last opened region.
 * <tt>token</tt> is itself a subclass of String, so you can use it just as you would a string. It represents the lexeme that was actually parsed.
 
-== Releasing Syntax
-
-* Update version in <tt>lib/syntax/version.rb</tt>.
-
-* Build the gem, use Ruby 1.9.3.
-
-    rake clean
-    rake package

data/lib/syntax/version.rb

@@ -1,7 +1,7 @@
 module Syntax
   module Version
     MAJOR=1
-    MINOR=1
+    MINOR=2
     TINY=0
 
     STRING=[MAJOR,MINOR,TINY].join('.')

metadata

@@ -1,22 +1,25 @@
 --- !ruby/object:Gem::Specification
 name: syntax
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
   prerelease: 
 platform: ruby
 authors:
 - Jamis Buck
-autorequire: syntax
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-11 00:00:00.000000000 Z
+date: 2014-01-02 00:00:00.000000000 Z
 dependencies: []
-description: 
+description: Syntax is Ruby library for performing simple syntax highlighting.
 email: jamis@jamisbuck.org
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- README.rdoc
+- CHANGELOG
 - lib/syntax/common.rb
 - lib/syntax/convertors/abstract.rb
 - lib/syntax/convertors/html.rb
@@ -30,17 +33,9 @@ files:
 - lib/syntax/lang/yaml.rb
 - lib/syntax/version.rb
 - lib/syntax.rb
-- test/ALL-TESTS.rb
-- test/syntax/tc_ruby.rb
-- test/syntax/tc_xml.rb
-- test/syntax/tc_yaml.rb
-- test/syntax/tokenizer_testcase.rb
-- test/tc_syntax.rb
-- README.rdoc
-- LICENSE
-- CHANGELOG
-homepage: 
-licenses: []
+homepage: https://github.com/dblock/syntax
+licenses:
+- BSD
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -62,6 +57,5 @@ rubyforge_project:
 rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
-summary: Syntax is Ruby library for performing simple syntax highlighting.
-test_files:
-- test/ALL-TESTS.rb
+summary: Perform simple syntax highlighting.
+test_files: []

data/test/ALL-TESTS.rb

@@ -1,5 +0,0 @@
-#!/usr/bin/env ruby
-$:.unshift "../lib"
-
-Dir.chdir File.dirname(__FILE__)
-Dir["**/tc_*.rb"].each { |file| load File.expand_path(file) }

data/test/syntax/tc_ruby.rb

@@ -1,871 +0,0 @@
-require File.dirname(__FILE__) + "/tokenizer_testcase"
-
-class TC_Syntax_Ruby < TokenizerTestCase
-
-  syntax "ruby"
-
-  def test_empty
-    tokenize ""
-    assert_no_next_token
-  end
-
-  def test_constant
-    tokenize "Foo"
-    assert_next_token :constant, "Foo"
-  end
-
-  def test_ident
-    tokenize "foo"
-    assert_next_token :ident, "foo"
-  end
-
-  def test_comment_eol
-    tokenize "# a comment\nfoo"
-    assert_next_token :comment, "# a comment"
-    assert_next_token :normal, "\n"
-    assert_next_token :ident, "foo"
-  end
-
-  def test_comment_block
-    tokenize "=begin\nthis is a comment\n=end\nnoncomment"
-    assert_next_token :comment, "=begin\nthis is a comment\n=end"
-    assert_next_token :normal, "\n"
-    assert_next_token :ident, "noncomment"
-  end
-
-  def test_comment_block_with_CRNL
-    tokenize "=begin\r\nthis is a comment\r\n=end\r\nnoncomment"
-    assert_next_token :comment, "=begin\r\nthis is a comment\r\n=end"
-    assert_next_token :normal, "\r\n"
-    assert_next_token :ident, "noncomment"
-  end
-
-  def test_keyword
-    Syntax::Ruby::KEYWORDS.each do |word|
-      tokenize word
-      assert_next_token :keyword, word
-    end
-    Syntax::Ruby::KEYWORDS.each do |word|
-      tokenize "foo.#{word}"
-      skip_token 2
-      assert_next_token :ident, word
-    end
-  end
-
-  def test__END__
-    tokenize "__END__\n\nblah blah blah"
-    assert_next_token :comment, "__END__\n\nblah blah blah"
-  end
-
-  def test__END__with_CRNL
-    tokenize "__END__\r\nblah blah blah"
-    assert_next_token :comment, "__END__\r\nblah blah blah"
-  end
-
-  def test_def_paren
-    tokenize "def  foo(bar)"
-    assert_next_token :keyword, "def  "
-    assert_next_token :method, "foo"
-    assert_next_token :punct, "("
-    assert_next_token :ident, "bar"
-    assert_next_token :punct, ")"
-  end
-
-  def test_def_space
-    tokenize "def  foo bar"
-    assert_next_token :keyword, "def  "
-    assert_next_token :method, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :ident, "bar"
-  end
-
-  def test_def_semicolon
-    tokenize "def  foo;"
-    assert_next_token :keyword, "def  "
-    assert_next_token :method, "foo"
-    assert_next_token :punct, ";"
-  end
-
-  def test_def_eol
-    tokenize "def foo"
-    assert_next_token :keyword, "def "
-    assert_next_token :method, "foo"
-  end
-
-  def test_class_space
-    tokenize "class    Foo\n"
-    assert_next_token :keyword, "class    "
-    assert_next_token :class, "Foo"
-    assert_next_token :normal, "\n"
-  end
-
-  def test_class_semicolon
-    tokenize "class    Foo;"
-    assert_next_token :keyword, "class    "
-    assert_next_token :class, "Foo"
-    assert_next_token :punct, ";"
-  end
-
-  def test_class_extend
-    tokenize "class    Foo< Bang"
-    assert_next_token :keyword, "class    "
-    assert_next_token :class, "Foo"
-    assert_next_token :punct, "<"
-    assert_next_token :normal, " "
-    assert_next_token :constant, "Bang"
-  end
-
-  def test_module_space
-    tokenize "module    Foo\n"
-    assert_next_token :keyword, "module    "
-    assert_next_token :module, "Foo"
-    assert_next_token :normal, "\n"
-  end
-
-  def test_module_semicolon
-    tokenize "module    Foo;"
-    assert_next_token :keyword, "module    "
-    assert_next_token :module, "Foo"
-    assert_next_token :punct, ";"
-  end
-
-  def test_module_other
-    tokenize "module    Foo!\n"
-    assert_next_token :keyword, "module    "
-    assert_next_token :module, "Foo!"
-  end
-
-  def test_scope_operator
-    tokenize "Foo::Bar"
-    assert_next_token :constant, "Foo"
-    assert_next_token :punct, "::"
-    assert_next_token :constant, "Bar"
-  end
-
-  def test_symbol_dquote
-    tokenize ':"foo"'
-    assert_next_token :symbol, ':"'
-    assert_next_token :symbol, '', :region_open
-    assert_next_token :symbol, 'foo'
-    assert_next_token :symbol, '', :region_close
-    assert_next_token :symbol, '"'
-    assert_no_next_token
-  end
-
-  def test_symbol_squote
-    tokenize ":'foo'"
-    assert_next_token :symbol, ":'"
-    assert_next_token :symbol, "", :region_open
-    assert_next_token :symbol, "foo"
-    assert_next_token :symbol, "", :region_close
-    assert_next_token :symbol, "'"
-    assert_no_next_token
-  end
-
-  def test_symbol
-    tokenize ":foo_123"
-    assert_next_token :symbol, ":foo_123"
-
-    tokenize ":123"
-    assert_next_token :punct, ":"
-    assert_next_token :number, "123"
-
-    tokenize ":foo="
-    assert_next_token :symbol, ":foo="
-
-    tokenize ":foo!"
-    assert_next_token :symbol, ":foo!"
-
-    tokenize ":foo?"
-    assert_next_token :symbol, ":foo?"
-  end
-
-  def test_char
-    tokenize "?."
-    assert_next_token :char, "?."
-
-    tokenize '?\n'
-    assert_next_token :char, '?\n'
-  end
-
-  def test_specials
-    %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tokenize word
-      assert_next_token :constant, word
-    end
-
-    %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tokenize "#{word}?"
-      assert_next_token :ident, "#{word}?"
-    end
-
-    %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tokenize "#{word}!"
-      assert_next_token :ident, "#{word}!"
-    end
-
-    %w{__FILE__ __LINE__ true false nil self}.each do |word|
-      tokenize "x.#{word}"
-      skip_token 2
-      assert_next_token :ident, word
-    end
-  end
-
-  def test_pct_r
-    tokenize '%r{foo#{x}bar}'
-    assert_next_token :punct, "%r{"
-    assert_next_token :regex, "", :region_open
-    assert_next_token :regex, "foo"
-    assert_next_token :expr, '#{x}'
-    assert_next_token :regex, "bar"
-    assert_next_token :regex, "", :region_close
-    assert_next_token :punct, "}"
-
-    tokenize '%r-foo#{x}bar-'
-    assert_next_token :punct, "%r-"
-    assert_next_token :regex, "", :region_open
-    assert_next_token :regex, "foo"
-    assert_next_token :expr, '#{x}'
-    assert_next_token :regex, "bar"
-    assert_next_token :regex, "", :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_r_with_wakas
-    tokenize '%r<foo#{x}bar> foo'
-    assert_next_token :punct, "%r<"
-    assert_next_token :regex, "", :region_open
-    assert_next_token :regex, "foo"
-    assert_next_token :expr, '#{x}'
-    assert_next_token :regex, "bar"
-    assert_next_token :regex, "", :region_close
-    assert_next_token :punct, ">"
-    assert_next_token :normal, " "
-    assert_next_token :ident, "foo"
-  end
-
-  def test_pct_w_brace
-    tokenize '%w{foo bar baz}'
-    assert_next_token :punct, "%w{"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'foo bar baz'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "}"
-  end
-
-  def test_pct_w
-    tokenize '%w-foo#{x} bar baz-'
-    assert_next_token :punct, "%w-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'foo#{x} bar baz'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_q
-    tokenize '%q-hello #{world}-'
-    assert_next_token :punct, "%q-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'hello #{world}'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
- 
-  def test_pct_s
-    tokenize '%s-hello #{world}-'
-    assert_next_token :punct, "%s-"
-    assert_next_token :symbol, '', :region_open
-    assert_next_token :symbol, 'hello #{world}'
-    assert_next_token :symbol, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_W
-    tokenize '%W-foo#{x} bar baz-'
-    assert_next_token :punct, "%W-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'foo'
-    assert_next_token :expr, '#{x}'
-    assert_next_token :string, ' bar baz'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_Q
-    tokenize '%Q-hello #{world}-'
-    assert_next_token :punct, "%Q-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'hello '
-    assert_next_token :expr, '#{world}'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_x
-    tokenize '%x-ls /blah/#{foo}-'
-    assert_next_token :punct, "%x-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'ls /blah/'
-    assert_next_token :expr, '#{foo}'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_pct_string
-    tokenize '%-hello #{world}-'
-    assert_next_token :punct, "%-"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'hello '
-    assert_next_token :expr, '#{world}'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, "-"
-  end
-
-  def test_bad_pct_string
-    tokenize '%0hello #{world}0'
-    assert_next_token :punct, "%"
-    assert_next_token :number, '0'
-    assert_next_token :ident, 'hello'
-    assert_next_token :normal, ' '
-    assert_next_token :comment, '#{world}0'
-  end
-
-  def test_shift_left
-    tokenize 'foo << 5'
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<"
-    assert_next_token :normal, " "
-    assert_next_token :number, "5"
-  end
-
-  def test_shift_left_no_white
-    tokenize 'foo<<5'
-    assert_next_token :ident, "foo"
-    assert_next_token :punct, "<<"
-    assert_next_token :number, "5"
-  end
-
-  def test_here_doc_no_opts
-    tokenize "foo <<EOF\n  foo\n  bar\n  baz\nEOF"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<"
-    assert_next_token :constant, "EOF"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  foo\n  bar\n  baz\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "EOF"
-  end
-
-  def test_here_doc_no_opts_missing_end
-    tokenize "foo <<EOF\n  foo\n  bar\n  baz\n EOF"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<"
-    assert_next_token :constant, "EOF"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  foo\n  bar\n  baz\n EOF"
-    assert_no_next_token
-  end
-
-  def test_here_doc_float_right
-    tokenize "foo <<-EOF\n  foo\n  bar\n  baz\n  EOF"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<-"
-    assert_next_token :constant, "EOF"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  foo\n  bar\n  baz\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "  EOF"
-  end
-
-  def test_here_doc_single_quotes
-    tokenize "foo <<'EOF'\n  foo\#{x}\n  bar\n  baz\nEOF"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<'"
-    assert_next_token :constant, "EOF"
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  foo\#{x}\n  bar\n  baz\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "EOF"
-  end
-
-  def test_here_doc_double_quotes
-    tokenize "foo <<\"EOF\"\n  foo\#{x}\n  bar\n  baz\nEOF"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<\""
-    assert_next_token :constant, "EOF"
-    assert_next_token :punct, "\""
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  foo"
-    assert_next_token :expr, '#{x}'
-    assert_next_token :string, "\n  bar\n  baz\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "EOF"
-  end
-
-  def test_space
-    tokenize "\n  \t\t\n\n\r\n"
-    assert_next_token :normal, "\n  \t\t\n\n\r\n"
-  end
-
-  def test_number
-    tokenize "1 1.0 1e5 1.0e5 1_2.5 1_2.5_2 1_2.5_2e3_2"
-    assert_next_token :number, "1"
-    skip_token
-    assert_next_token :number, "1.0"
-    skip_token
-    assert_next_token :number, "1e5"
-    skip_token
-    assert_next_token :number, "1.0e5"
-    skip_token
-    assert_next_token :number, "1_2.5"
-    skip_token
-    assert_next_token :number, "1_2.5_2"
-    skip_token
-    assert_next_token :number, "1_2.5_2e3_2"
-  end
-
-  def test_dquoted_string
-    tokenize '"foo #{x} bar\"\n\tbaz\xA5b\5\1234"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'foo '
-    assert_next_token :expr, '#{x}'
-    assert_next_token :string, ' bar'
-    assert_next_token :escape, '\"\n\t'
-    assert_next_token :string, 'baz'
-    assert_next_token :escape, '\xA5'
-    assert_next_token :string, 'b'
-    assert_next_token :escape, '\5\123'
-    assert_next_token :string, '4'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_squoted_string
-    tokenize '\'foo #{x} bar\\\'\n\tbaz\\\\\xA5b\5\1234\''
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, 'foo #{x} bar'
-    assert_next_token :escape, '\\\''
-    assert_next_token :string, '\n\tbaz'
-    assert_next_token :escape, '\\\\'
-    assert_next_token :string, '\xA5b\5\1234'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, "'"
-  end
-
-  def test_dot_selector
-    tokenize 'foo.nil'
-    skip_token
-    assert_next_token :punct, "."
-    assert_next_token :ident, "nil"
-  end
-
-  def test_dot_range_inclusive
-    tokenize 'foo..nil'
-    skip_token
-    assert_next_token :punct, ".."
-    assert_next_token :constant, "nil"
-  end
-
-  def test_dot_range_exclusive
-    tokenize 'foo...nil'
-    skip_token
-    assert_next_token :punct, "..."
-    assert_next_token :constant, "nil"
-  end
-
-  def test_dot_range_many
-    tokenize 'foo.....nil'
-    skip_token
-    assert_next_token :punct, "....."
-    assert_next_token :constant, "nil"
-  end
-
-  def test_attribute
-    tokenize '@var_foo'
-    assert_next_token :attribute, "@var_foo"
-  end
-
-  def test_global
-    tokenize '$var_foo'
-    assert_next_token :global, "$var_foo"
-    tokenize '$12'
-    assert_next_token :global, "$12"
-    tokenize '$/f'
-    assert_next_token :global, "$/"
-    tokenize "$\n"
-    assert_next_token :global, "$"
-  end
-
-  def test_paren_delimiter
-    tokenize '%w(a)'
-    assert_next_token :punct, "%w("
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "a"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, ")"
-  end
-
-  def test_division
-    tokenize 'm / 3'
-    assert_next_token :ident, "m"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "/"
-    assert_next_token :normal, " "
-    assert_next_token :number, "3"
-  end
-
-  def test_regex
-    tokenize 'm =~ /3/'
-    assert_next_token :ident, "m"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "=~"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "/"
-    assert_next_token :regex, "", :region_open
-    assert_next_token :regex, "3"
-    assert_next_token :regex, "", :region_close
-    assert_next_token :punct, "/"
-  end
-
-  def test_heredoc_with_trailing_text
-    tokenize "foo('here', <<EOF)\n  A heredoc.\nEOF\nfoo"
-    assert_next_token :ident,  "foo"
-    assert_next_token :punct,  "('"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'here'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct,  "',"
-    assert_next_token :normal, ' '
-    assert_next_token :punct,  '<<'
-    assert_next_token :constant, "EOF"
-    assert_next_token :punct,  ')'
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n  A heredoc.\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "EOF"
-    assert_next_token :normal, "\n"
-    assert_next_token :ident,  "foo"
-  end
-
-  def test_multiple_heredocs
-    tokenize <<'TEST'
-foo('here', <<EOF, 'there', <<-'FOO', 'blah')
-First heredoc, right here.
-Expressions are #{allowed}
-EOF
-    Another heredoc, immediately after the first.
-    Expressions are not #{allowed}
-  FOO
-TEST
-    assert_next_token :ident,  "foo"
-    assert_next_token :punct,  "('"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'here'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct,  "',"
-    assert_next_token :normal, ' '
-    assert_next_token :punct,  '<<'
-    assert_next_token :constant, "EOF"
-    assert_next_token :punct,  ','
-    assert_next_token :normal, ' '
-    assert_next_token :punct,  "'"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'there'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct,  "',"
-    assert_next_token :normal, ' '
-    assert_next_token :punct,  "<<-'"
-    assert_next_token :constant, "FOO"
-    assert_next_token :punct,  "',"
-    assert_next_token :normal, ' '
-    assert_next_token :punct,  "'"
-    assert_next_token :string, '', :region_open
-    assert_next_token :string, 'blah'
-    assert_next_token :string, '', :region_close
-    assert_next_token :punct,  "')"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\nFirst heredoc, right here.\nExpressions are "
-    assert_next_token :expr, '#{allowed}'
-    assert_next_token :string, "\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "EOF"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\n    Another heredoc, immediately after the first.\n    Expressions are not \#{allowed}\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "  FOO"
-  end
-
-  def test_carldr_bad_heredoc_001
-    tokenize <<'TEST'
-str = <<END
-here document #{1 + 1}
-END
-
-if str
-TEST
-
-    assert_next_token :ident, "str"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "="
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<"
-    assert_next_token :constant, "END"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\nhere document "
-    assert_next_token :expr, '#{1 + 1}'
-    assert_next_token :string, "\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "END"
-    assert_next_token :normal, "\n\n"
-    assert_next_token :keyword, "if"
-    assert_next_token :normal, " "
-    assert_next_token :ident, "str"
-  end
-
-  def test_regex_after_keyword
-    tokenize "when /[0-7]/\nfoo"
-    assert_next_token :keyword, "when"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "/"
-    assert_next_token :regex, "", :region_open
-    assert_next_token :regex, "[0-7]"
-    assert_next_token :regex, "", :region_close
-    assert_next_token :punct, "/"
-    assert_next_token :normal, "\n"
-    assert_next_token :ident, "foo"
-  end
-
-  def test_heredoc_after_lparen
-    tokenize "foo(<<SRC, obj)\nblah blah\nSRC\nfoo"
-    assert_next_token :ident, "foo"
-    assert_next_token :punct, "(<<"
-    assert_next_token :constant, "SRC"
-    assert_next_token :punct, ","
-    assert_next_token :normal, " "
-    assert_next_token :ident, "obj"
-    assert_next_token :punct, ")"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\nblah blah\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "SRC"
-    assert_next_token :normal, "\n"
-    assert_next_token :ident, "foo"
-  end
-
-  def test_division_after_parens
-    tokenize "(a+b)/2"
-    assert_next_token :punct, "("
-    assert_next_token :ident, "a"
-    assert_next_token :punct, "+"
-    assert_next_token :ident, "b"
-    assert_next_token :punct, ")/"
-    assert_next_token :number, "2"
-  end
-
-  def test_heredoc_with_CRNL
-    tokenize "foo <<SRC\r\nSome text\r\nSRC\r\nfoo"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "<<"
-    assert_next_token :constant, "SRC"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "\r\nSome text\r\n"
-    assert_next_token :string, "", :region_close
-    assert_next_token :constant, "SRC"
-    assert_next_token :normal, "\r\n"
-    assert_next_token :ident, "foo"
-  end
-
-  def test_question_mark_at_newline
-    tokenize "foo ?\n 'bar': 'baz'"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "?"
-    assert_next_token :normal, "\n "
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "bar"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, "':"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "baz"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, "'"
-  end
-
-  def test_question_mark_and_escaped_newline
-    tokenize "foo ?\\\n 'bar': 'baz'"
-    assert_next_token :ident, "foo"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "?\\"
-    assert_next_token :normal, "\n "
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "bar"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, "':"
-    assert_next_token :normal, " "
-    assert_next_token :punct, "'"
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "baz"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, "'"
-  end
-
-  def test_highlighted_subexpression
-    tokenizer.set :expressions => :highlight
-    tokenize '"la la #{["hello", "world"].each { |f| puts "string #{f}" }}"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "la la "
-    assert_next_token :expr, "", :region_open
-    assert_next_token :expr, '#{'
-    assert_next_token :punct, '["'
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, 'hello'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '",'
-    assert_next_token :normal, ' '
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "world"
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"].'
-    assert_next_token :ident, 'each'
-    assert_next_token :normal, ' '
-    assert_next_token :punct, '{'
-    assert_next_token :normal, ' '
-    assert_next_token :punct, '|'
-    assert_next_token :ident, 'f'
-    assert_next_token :punct, '|'
-    assert_next_token :normal, ' '
-    assert_next_token :ident, 'puts'
-    assert_next_token :normal, ' '
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :string, "string "
-    assert_next_token :expr, "", :region_open
-    assert_next_token :expr, '#{'
-    assert_next_token :ident, 'f'
-    assert_next_token :expr, '}'
-    assert_next_token :expr, "", :region_close
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-    assert_next_token :normal, ' '
-    assert_next_token :punct, '}'
-    assert_next_token :expr, '}'
-    assert_next_token :expr, "", :region_close
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_expr_in_braces
-    tokenize '"#{f}"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :expr, '#{f}'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_expr_in_braces_with_nested_braces
-    tokenize '"#{loop{break}}"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :expr, '#{loop{break}}'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_expr_with_global_var
-    tokenize '"#$f"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :expr, '#$f'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_expr_with_instance_var
-    tokenize '"#@f"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :expr, '#@f'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_expr_with_class_var
-    tokenize '"#@@f"'
-    assert_next_token :punct, '"'
-    assert_next_token :string, "", :region_open
-    assert_next_token :expr, '#@@f'
-    assert_next_token :string, "", :region_close
-    assert_next_token :punct, '"'
-  end
-
-  def test_qmark_space
-    tokenize "? "
-    assert_next_token :punct, "?"
-    assert_next_token :normal, " "
-  end
-
-  def test_capitalized_method
-    tokenize "obj.Foo"
-    skip_token 2
-    assert_next_token :ident, "Foo"
-  end
-
-  def test_hexadecimal_literal
-    tokenize "0xDEADbeef 0X1234567890ABCDEFG"
-    assert_next_token :number, "0xDEADbeef"
-    skip_token
-    assert_next_token :number, "0X1234567890ABCDEF"
-    assert_next_token :constant, "G"
-  end
-
-  def test_binary_literal
-    tokenize "0b2 0b0 0b101 0B123"
-    assert_next_token :number, "0"
-    assert_next_token :ident, "b2"
-    skip_token
-    assert_next_token :number, "0b0"
-    skip_token
-    assert_next_token :number, "0b101"
-    skip_token
-    assert_next_token :number, "0B123"
-  end
-
-  def test_octal_literal
-    tokenize "0o9 0o12345670abc 0O12345678"
-    assert_next_token :number, "0"
-    assert_next_token :ident, "o9"
-    skip_token
-    assert_next_token :number, "0o12345670"
-    assert_next_token :ident, "abc"
-    skip_token
-    assert_next_token :number, "0O12345678"
-  end
-
-  def test_decimal_literal
-    tokenize "0dA 0d1234567890abc 0D1234567890"
-    assert_next_token :number, "0"
-    assert_next_token :ident, "dA"
-    skip_token
-    assert_next_token :number, "0d1234567890"
-    assert_next_token :ident, "abc"
-    skip_token
-    assert_next_token :number, "0D1234567890"
-  end
-end