syntax 0.5.0

data/lib/syntax/xml.rb

@@ -0,0 +1,108 @@
+require 'syntax'
+
+module Syntax
+
+  # A simple implementation of an XML lexer. It handles most cases. It is
+  # not a validating lexer, meaning it will happily process invalid XML without
+  # complaining.
+  class XML < Tokenizer
+
+    # Initialize the lexer.
+    def setup
+      @in_tag = false
+    end
+
+    # Step through a single iteration of the tokenization process. This will
+    # yield (potentially) many tokens, and possibly zero tokens.
+    def step
+      start_group :normal, matched if scan( /\s+/ )
+      if @in_tag
+        case
+          when scan( /([-\w]+):([-\w]+)/ )
+            start_group :namespace, subgroup(1)
+            start_group :punct, ":"
+            start_group :attribute, subgroup(2)
+          when scan( /\d+/ )
+            start_group :number, matched
+          when scan( /[-\w]+/ )
+            start_group :attribute, matched
+          when scan( %r{[/?]?>} )
+            @in_tag = false
+            start_group :punct, matched
+          when scan( /=/ )
+            start_group :punct, matched
+          when scan( /["']/ )
+            scan_string matched
+          else
+            append getch
+        end
+      elsif ( text = scan_until( /(?=[<&])/ ) )
+        start_group :normal, text unless text.empty?
+        if scan(/<!--.*?(-->|\Z)/m)
+          start_group :comment, matched
+        else
+          case peek(1)
+            when "<"
+              start_group :punct, getch
+              case peek(1)
+                when "?"
+                  append getch
+                when "/"
+                  append getch
+                when "!"
+                  append getch
+              end
+              start_group :normal, matched if scan( /\s+/ )
+              if scan( /([-\w]+):([-\w]+)/ )
+                start_group :namespace, subgroup(1)
+                start_group :punct, ":"
+                start_group :tag, subgroup(2)
+              elsif scan( /[-\w]+/ )
+                start_group :tag, matched
+              end
+              @in_tag = true
+            when "&"
+              if scan( /&\S{1,10};/ )
+                start_group :entity, matched
+              else
+                start_group :normal, scan( /&/ )
+              end
+          end
+        end
+      else
+        append scan_until( /\Z/ )
+      end
+    end
+
+    private
+
+      # Scan the string starting at the current position, with the given
+      # delimiter character.
+      def scan_string( delim )
+        start_group :punct, delim
+        match = /(?=[&\\]|#{delim})/
+        loop do
+          break unless ( text = scan_until( match ) )
+          start_group :string, text unless text.empty?
+          case peek(1)
+            when "&"
+              if scan( /&\S{1,10};/ )
+                start_group :entity, matched
+              else
+                start_group :string, getch
+              end
+            when "\\"
+              start_group :string, getch
+              append getch || ""
+            when delim
+              start_group :punct, getch
+              break
+          end
+        end
+      end
+
+  end
+
+  SYNTAX["xml"] = XML
+
+end

data/lib/syntax/yaml.rb

@@ -0,0 +1,105 @@
+require 'syntax'
+
+module Syntax
+
+  # A simple implementation of an YAML lexer. It handles most cases. It is
+  # not a validating lexer.
+  class YAML < Tokenizer
+
+    # Step through a single iteration of the tokenization process. This will
+    # yield (potentially) many tokens, and possibly zero tokens.
+    def step
+      if bol?
+        case
+          when scan(/---(\s*.+)?$/)
+            start_group :document, matched
+          when scan(/(\s*)([a-zA-Z][-\w]*)(\s*):/)
+            start_group :normal, subgroup(1)
+            start_group :key, subgroup(2)
+            start_group :normal, subgroup(3)
+            start_group :punct, ":"
+          when scan(/(\s*)-/)
+            start_group :normal, subgroup(1)
+            start_group :punct, "-"
+          when scan(/\s*$/)
+            start_group :normal, matched
+          when scan(/#.*$/)
+            start_group :comment, matched
+          else
+            append getch
+        end
+      else
+        case
+          when scan(/[\n\r]+/)
+            start_group :normal, matched
+          when scan(/[ \t]+/)
+            start_group :normal, matched
+          when scan(/!+(.*?^)?\S+/)
+            start_group :type, matched
+          when scan(/&\S+/)
+            start_group :anchor, matched
+          when scan(/\*\S+/)
+            start_group :ref, matched
+          when scan(/\d\d:\d\d:\d\d/)
+            start_group :time, matched
+          when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
+            start_group :date, matched
+          when scan(/['"]/)
+            start_group :punct, matched
+            scan_string matched
+          when scan(/:\w+/)
+            start_group :symbol, matched
+          when scan(/[:]/)
+            start_group :punct, matched
+          when scan(/#.*$/)
+            start_group :comment, matched
+          when scan(/>-?/)
+            start_group :punct, matched
+            start_group :normal, scan(/.*$/)
+            append getch until eos? || bol?
+            return if eos?
+            indent = check(/ */)
+            start_group :string
+            loop do
+              line = check_until(/[\n\r]|\Z/)
+              break if line.nil?
+              if line.chomp.length > 0
+                this_indent = line.chomp.match( /^\s*/ )[0]
+                break if this_indent.length < indent.length
+              end
+              append scan_until(/[\n\r]|\Z/)
+            end
+          else
+            start_group :normal, scan_until(/(?=$|#)/)
+        end
+      end
+    end
+
+    private
+
+      def scan_string( delim )
+        regex = /(?=[#{delim=="'" ? "" : "\\\\"}#{delim}])/
+        loop do
+          text = scan_until( regex )
+          if text.nil?
+            start_group :string, scan_until( /\Z/ )
+            break
+          else
+            start_group :string, text unless text.empty?
+          end
+
+          case peek(1)
+            when "\\"
+              start_group :expr, scan(/../)
+            else
+              start_group :punct, getch
+              break
+          end
+        end
+      end
+
+  end
+
+  SYNTAX["yaml"] = YAML
+
+end

data/test/ALL-TESTS.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$:.unshift "../lib"
+
+Dir.chdir File.dirname(__FILE__)
+Dir["**/tc_*.rb"].each { |file| load file }

data/test/syntax/tc_ruby.rb

@@ -0,0 +1,518 @@
+$:.unshift "../../lib"
+
+require 'test/unit'
+require 'syntax/ruby'
+
+class TC_Syntax_Ruby < Test::Unit::TestCase
+
+  def setup
+    @ruby = Syntax::Ruby.new
+  end
+
+  def test_empty
+    called = false
+    @ruby.tokenize( "" ) { |tok| called = true }
+    assert !called
+  end
+
+  def test_constant
+    called = false
+    @ruby.tokenize( "Foo" ) do |tok|
+      called = true
+      assert_equal :constant, tok.group
+      assert_equal "Foo", tok
+    end
+    assert called
+  end
+
+  def test_ident
+    called = false
+    @ruby.tokenize( "foo" ) do |tok|
+      called = true
+      assert_equal :ident, tok.group
+      assert_equal "foo", tok
+    end
+    assert called
+  end
+
+  def test_comment_eol
+    called = false
+    @ruby.tokenize( "# a comment" ) do |tok|
+      called = true
+      assert_equal :comment, tok.group
+      assert_equal "# a comment", tok
+    end
+    assert called
+  end
+
+  def test_comment_block
+    called = false
+    @ruby.tokenize( "=begin\nthis is a comment\n=end" ) do |tok|
+      called = true
+      assert_equal :comment, tok.group
+      assert_equal "=begin\nthis is a comment\n=end", tok
+    end
+    assert called
+  end
+
+  def test_keyword
+    Syntax::Ruby::KEYWORDS.each do |word|
+      tok = []
+      @ruby.tokenize( word ) { |t| tok << t }
+      assert_equal [ :keyword, word ], [ tok.first.group, tok.first ]
+    end
+    Syntax::Ruby::KEYWORDS.each do |word|
+      tok = []
+      @ruby.tokenize( "foo.#{word}" ) { |t| tok << t }
+      tok.shift
+      tok.shift
+      assert_equal [ :ident, word ], [ tok.first.group, tok.first ]
+    end
+  end
+
+  def test__END__
+    called = false
+    @ruby.tokenize( "__END__\n\nblah blah blah" ) do |tok|
+      called = true
+      assert_equal :comment, tok.group
+      assert_equal "__END__\n\nblah blah blah", tok
+    end
+    assert called
+  end
+
+  def test_def_paren
+    tok = nil
+    @ruby.start( "def  foo(bar)" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "def  ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "foo", tok
+    assert_equal :method, tok.group
+  end
+
+  def test_def_space
+    tok = nil
+    @ruby.start( "def  foo bar" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "def  ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "foo", tok
+    assert_equal :method, tok.group
+  end
+
+  def test_def_semicolon
+    tok = nil
+    @ruby.start( "def  foo;" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "def  ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "foo", tok
+    assert_equal :method, tok.group
+  end
+
+  def test_class_space
+    tok = nil
+    @ruby.start( "class    Foo\n" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "class    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo", tok
+    assert_equal :class, tok.group
+  end
+
+  def test_class_semicolon
+    tok = nil
+    @ruby.start( "class    Foo;" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "class    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo", tok
+    assert_equal :class, tok.group
+  end
+
+  def test_class_extend
+    tok = nil
+    @ruby.start( "class    Foo< Bang" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "class    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo", tok
+    assert_equal :class, tok.group
+  end
+
+  def test_module_space
+    tok = nil
+    @ruby.start( "module    Foo\n" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "module    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo", tok
+    assert_equal :module, tok.group
+  end
+
+  def test_module_semicolon
+    tok = nil
+    @ruby.start( "module    Foo;" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "module    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo", tok
+    assert_equal :module, tok.group
+  end
+
+  def test_module_other
+    tok = nil
+    @ruby.start( "module    Foo!\n" ) { |t| tok = t }
+    @ruby.step
+    assert_equal "module    ", tok
+    assert_equal :keyword, tok.group
+    @ruby.step
+    assert_equal "Foo!", tok
+    assert_equal :module, tok.group
+  end
+
+  def test_scope_operator
+    tok = []
+    @ruby.tokenize( "Foo::Bar" ) { |t| tok << t }
+    assert_equal "Foo", tok.shift
+    assert_equal "::", tok.first
+    assert_equal :punct, tok.shift.group
+    assert_equal "Bar", tok.first
+  end
+
+  def test_symbol_dquote
+    tok = []
+    @ruby.tokenize( ':"foo"' ) { |t| tok << t }
+    assert_equal ':"foo"', tok.first
+    assert_equal :symbol, tok.first.group
+  end
+
+  def test_symbol_squote
+    tok = []
+    @ruby.tokenize( ":'foo'" ) { |t| tok << t }
+    assert_equal ":'foo'", tok.first
+    assert_equal :symbol, tok.first.group
+  end
+
+  def test_symbol
+    tok = []
+    @ruby.tokenize( ":foo_bar?" ) { |t| tok << t }
+    assert_equal ":foo_bar?", tok.first
+    assert_equal :symbol, tok.first.group
+  end
+
+  def test_char
+    tok = []
+    @ruby.tokenize( "?." ) { |t| tok << t }
+    assert_equal "?.", tok.first
+    assert_equal :char, tok.first.group
+
+    tok.clear
+    @ruby.tokenize( '?\n' ) { |t| tok << t }
+    assert_equal '?\n', tok.first
+    assert_equal :char, tok.first.group
+  end
+
+  def test_specials
+    %w{__FILE__ __LINE__ true false nil self}.each do |word|
+      tok = []
+      @ruby.tokenize( word ) { |t| tok << t }
+      assert_equal word, tok.first
+      assert_equal :constant, tok.first.group
+    end
+
+    %w{__FILE__ __LINE__ true false nil self}.each do |word|
+      tok = []
+      @ruby.tokenize( "#{word}?" ) { |t| tok << t }
+      assert_equal "#{word}?", tok.first
+      assert_equal :ident, tok.first.group
+    end
+
+    %w{__FILE__ __LINE__ true false nil self}.each do |word|
+      tok = []
+      @ruby.tokenize( "#{word}!" ) { |t| tok << t }
+      assert_equal "#{word}!", tok.first
+      assert_equal :ident, tok.first.group
+    end
+
+    %w{__FILE__ __LINE__ true false nil self}.each do |word|
+      tok = []
+      @ruby.tokenize( "x.#{word}" ) { |t| tok << t }
+      tok.shift
+      tok.shift
+      assert_equal word, tok.first
+      assert_equal :ident, tok.first.group
+    end
+  end
+
+  def test_pct_r
+    tok = []
+    @ruby.tokenize( '%r{foo#{x}bar}' ) { |t| tok << t }
+    assert_equal [ :punct, "%r{" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "}" ], [ tok.first.group, tok.shift ]
+
+    tok = []
+    @ruby.tokenize( '%r-foo#{x}bar-' ) { |t| tok << t }
+    assert_equal [ :punct, "%r-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :regex, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :regex, "bar" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_w
+    tok = []
+    @ruby.tokenize( '%w-foo#{x} bar baz-' ) { |t| tok << t }
+    assert_equal [ :punct, "%w-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'foo#{x} bar baz' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_q
+    tok = []
+    @ruby.tokenize( '%q-hello #{world}-' ) { |t| tok << t }
+    assert_equal [ :punct, "%q-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'hello #{world}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+ 
+  def test_pct_s
+    tok = []
+    @ruby.tokenize( '%s-hello #{world}-' ) { |t| tok << t }
+    assert_equal [ :punct, "%s-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :symbol, 'hello #{world}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_W
+    tok = []
+    @ruby.tokenize( '%W-foo#{x} bar baz-' ) { |t| tok << t }
+    assert_equal [ :punct, "%W-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'foo' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, ' bar baz' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_Q
+    tok = []
+    @ruby.tokenize( '%Q-hello #{world}-' ) { |t| tok << t }
+    assert_equal [ :punct, "%Q-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_x
+    tok = []
+    @ruby.tokenize( '%x-ls /blah/#{foo}-' ) { |t| tok << t }
+    assert_equal [ :punct, "%x-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'ls /blah/' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{foo}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_pct_string
+    tok = []
+    @ruby.tokenize( '%-hello #{world}-' ) { |t| tok << t }
+    assert_equal [ :punct, "%-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'hello ' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{world}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "-" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_bad_pct_string
+    tok = []
+    @ruby.tokenize( '%0hello #{world}0' ) { |t| tok << t }
+    assert_equal [ :punct, "%" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :number, '0' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :ident, 'hello' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, ' ' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :comment, '#{world}0' ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_shift_left
+    tok = []
+    @ruby.tokenize( 'foo << 5' ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :number, "5" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_here_doc_no_opts
+    tok = []
+    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\nEOF" ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_here_doc_no_opts_missing_end
+    tok = []
+    @ruby.tokenize( "foo <<EOF\n  foo\n  bar\n  baz\n EOF" ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  foo\n  bar\n  baz\n EOF" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_here_doc_float_right
+    tok = []
+    @ruby.tokenize( "foo <<-EOF\n  foo\n  bar\n  baz\n  EOF" ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<-" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  foo\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "  EOF" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_here_doc_single_quotes
+    tok = []
+    @ruby.tokenize( "foo <<'EOF'\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  foo\#{x}\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_here_doc_double_quotes
+    tok = []
+    @ruby.tokenize( "foo <<\"EOF\"\n  foo\#{x}\n  bar\n  baz\nEOF" ) { |t| tok << t }
+    assert_equal [ :ident, "foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :normal, " " ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "<<\"" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "\"" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  foo" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, "\n  bar\n  baz\n" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "EOF" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_space
+    tok = []
+    @ruby.tokenize( "\n  \t\t\n\n\r\n" ) { |t| tok << t }
+    assert_equal [ :normal, "\n  \t\t\n\n\r\n" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_number
+    tok = []
+    @ruby.tokenize( "1 1.0 1e5 1.0e5 1_2.5 1_2.5_2 1_2.5_2e3_2" ) { |t| tok << t }
+    assert_equal [ :number, "1" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1.0" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1e5" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1.0e5" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1_2.5" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1_2.5_2" ], [ tok.first.group, tok.shift ]
+    tok.shift
+    assert_equal [ :number, "1_2.5_2e3_2" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_dquoted_string
+    tok = []
+    @ruby.tokenize( '"foo #{x} bar\"\n\tbaz\xA5b\5\1234"' ) { |t| tok << t }
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'foo ' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '#{x}' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, ' bar' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\"\n\t' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'baz' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\xA5' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'b' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\5\123' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, '4' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, '"' ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_squoted_string
+    tok = []
+    @ruby.tokenize( '\'foo #{x} bar\\\'\n\tbaz\\\\\xA5b\5\1234\'' ) { |t| tok << t }
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, 'foo #{x} bar' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\\\'' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, '\n\tbaz' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :expr, '\\\\' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :string, '\xA5b\5\1234' ], [ tok.first.group, tok.shift ]
+    assert_equal [ :punct, "'" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_dot_selector
+    tok = []
+    @ruby.tokenize( 'foo.nil' ) { |t| tok << t }
+    tok.shift
+    assert_equal [ :punct, "." ], [ tok.first.group, tok.shift ]
+    assert_equal [ :ident, "nil" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_dot_range_inclusive
+    tok = []
+    @ruby.tokenize( 'foo..nil' ) { |t| tok << t }
+    tok.shift
+    assert_equal [ :punct, ".." ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_dot_range_exclusive
+    tok = []
+    @ruby.tokenize( 'foo...nil' ) { |t| tok << t }
+    tok.shift
+    assert_equal [ :punct, "..." ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_dot_range_many
+    tok = []
+    @ruby.tokenize( 'foo.....nil' ) { |t| tok << t }
+    tok.shift
+    assert_equal [ :punct, "....." ], [ tok.first.group, tok.shift ]
+    assert_equal [ :constant, "nil" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_attribute
+    tok = []
+    @ruby.tokenize( '@var_foo' ) { |t| tok << t }
+    assert_equal [ :attribute, "@var_foo" ], [ tok.first.group, tok.shift ]
+  end
+
+  def test_global
+    tok = []
+    @ruby.tokenize( '$var_foo' ) { |t| tok << t }
+    assert_equal [ :global, "$var_foo" ], [ tok.first.group, tok.shift ]
+    tok = []
+    @ruby.tokenize( '$12' ) { |t| tok << t }
+    assert_equal [ :global, "$12" ], [ tok.first.group, tok.shift ]
+    tok = []
+    @ruby.tokenize( '$/f' ) { |t| tok << t }
+    assert_equal [ :global, "$/" ], [ tok.first.group, tok.shift ]
+    tok = []
+    @ruby.tokenize( "$\n" ) { |t| tok << t }
+    assert_equal [ :global, "$" ], [ tok.first.group, tok.shift ]
+  end
+
+end