coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/php.rb

@@ -3,14 +3,19 @@ module Scanners
   
   load :html
   
+  # Scanner for PHP.
+  # 
   # Original by Stefan Walk.
   class PHP < Scanner
     
     register_for :php
     file_extension 'php'
+    encoding 'BINARY'
     
     KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
     
+  protected
+    
     def setup
       @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
     end
@@ -20,7 +25,7 @@ module Scanners
       @html_scanner.reset
     end
     
-    module Words
+    module Words  # :nodoc:
       
       # according to http://www.php.net/manual/en/reserved.keywords.php
       KEYWORDS = %w[
@@ -176,20 +181,20 @@ module Scanners
         $argc $argv
       ]
       
-      IDENT_KIND = CaseIgnoringWordList.new(:ident).
-        add(KEYWORDS, :reserved).
-        add(TYPES, :pre_type).
-        add(LANGUAGE_CONSTRUCTS, :reserved).
+      IDENT_KIND = WordList::CaseIgnoring.new(:ident).
+        add(KEYWORDS, :keyword).
+        add(TYPES, :predefined_type).
+        add(LANGUAGE_CONSTRUCTS, :keyword).
         add(BUILTIN_FUNCTIONS, :predefined).
-        add(CLASSES, :pre_constant).
+        add(CLASSES, :predefined_constant).
         add(EXCEPTIONS, :exception).
-        add(CONSTANTS, :pre_constant)
+        add(CONSTANTS, :predefined_constant)
       
       VARIABLE_KIND = WordList.new(:local_variable).
         add(PREDEFINED, :predefined)
     end
     
-    module RE
+    module RE  # :nodoc:
       
       PHP_START = /
         <script\s+[^>]*?language\s*=\s*"php"[^>]*?> |
@@ -224,17 +229,13 @@ module Scanners
       
     end
     
-    def scan_tokens tokens, options
-      if string.respond_to?(:encoding)
-        unless string.encoding == Encoding::ASCII_8BIT
-          self.string = string.encode Encoding::ASCII_8BIT,
-            :invalid => :replace, :undef => :replace, :replace => '?'
-        end
-      end
+  protected
+    
+    def scan_tokens encoder, options
       
       if check(RE::PHP_START) ||  # starts with <?
-       (match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
-       exist?(RE::HTML_INDICATOR) ||
+       (match?(/\s*<\S/) && check(/.{1,1000}#{RE::PHP_START}/om)) || # starts with tag and contains <?
+       check(/.{0,1000}#{RE::HTML_INDICATOR}/om) ||
        check(/.{1,100}#{RE::PHP_START}/om)  # PHP start after max 100 chars
         # is HTML with embedded PHP, so start with HTML
         states = [:initial]
@@ -252,29 +253,24 @@ module Scanners
       
       until eos?
         
-        match = nil
-        kind = nil
-        
         case states.last
         
         when :initial  # HTML
-          if scan RE::PHP_START
-            kind = :inline_delimiter
+          if match = scan(RE::PHP_START)
+            encoder.text_token match, :inline_delimiter
             label_expected = true
             states << :php
           else
             match = scan_until(/(?=#{RE::PHP_START})/o) || scan_rest
             @html_scanner.tokenize match unless match.empty?
-            next
           end
         
         when :php
           if match = scan(/\s+/)
-            tokens << [match, :space]
-            next
+            encoder.text_token match, :space
           
-          elsif scan(%r! (?m: \/\* (?: .*? \*\/ | .* ) ) | (?://|\#) .*? (?=#{RE::PHP_END}|$) !xo)
-            kind = :comment
+          elsif match = scan(%r! (?m: \/\* (?: .*? \*\/ | .* ) ) | (?://|\#) .*? (?=#{RE::PHP_END}|$) !xo)
+            encoder.text_token match, :comment
           
           elsif match = scan(RE::IDENTIFIER)
             kind = Words::IDENT_KIND[match]
@@ -285,7 +281,7 @@ module Scanners
               label_expected = false
               if kind == :ident && match =~ /^[A-Z]/
                 kind = :constant
-              elsif kind == :reserved
+              elsif kind == :keyword
                 case match
                 when 'class'
                   states << :class_expected
@@ -299,77 +295,68 @@ module Scanners
                 next
               end
             end
+            encoder.text_token match, kind
           
-          elsif scan(/(?:\d+\.\d*|\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+/i)
+          elsif match = scan(/(?:\d+\.\d*|\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+/i)
             label_expected = false
-            kind = :float
+            encoder.text_token match, :float
           
-          elsif scan(/0x[0-9a-fA-F]+/)
+          elsif match = scan(/0x[0-9a-fA-F]+/)
             label_expected = false
-            kind = :hex
+            encoder.text_token match, :hex
           
-          elsif scan(/\d+/)
+          elsif match = scan(/\d+/)
             label_expected = false
-            kind = :integer
-          
-          elsif scan(/'/)
-            tokens << [:open, :string]
-            if modifier
-              tokens << [modifier, :modifier]
-              modifier = nil
-            end
-            kind = :delimiter
-            states.push :sqstring
+            encoder.text_token match, :integer
           
-          elsif match = scan(/["`]/)
-            tokens << [:open, :string]
+          elsif match = scan(/['"`]/)
+            encoder.begin_group :string
             if modifier
-              tokens << [modifier, :modifier]
+              encoder.text_token modifier, :modifier
               modifier = nil
             end
             delimiter = match
-            kind = :delimiter
-            states.push :dqstring
+            encoder.text_token match, :delimiter
+            states.push match == "'" ? :sqstring : :dqstring
           
           elsif match = scan(RE::VARIABLE)
             label_expected = false
-            kind = Words::VARIABLE_KIND[match]
+            encoder.text_token match, Words::VARIABLE_KIND[match]
           
-          elsif scan(/\{/)
-            kind = :operator
+          elsif match = scan(/\{/)
+            encoder.text_token match, :operator
             label_expected = true
             states.push :php
           
-          elsif scan(/\}/)
+          elsif match = scan(/\}/)
             if states.size == 1
-              kind = :error
+              encoder.text_token match, :error
             else
               states.pop
               if states.last.is_a?(::Array)
                 delimiter = states.last[1]
                 states[-1] = states.last[0]
-                tokens << [matched, :delimiter]
-                tokens << [:close, :inline]
-                next
+                encoder.text_token match, :delimiter
+                encoder.end_group :inline
               else
-                kind = :operator
+                encoder.text_token match, :operator
                 label_expected = true
               end
             end
           
-          elsif scan(/@/)
+          elsif match = scan(/@/)
             label_expected = false
-            kind = :exception
+            encoder.text_token match, :exception
           
-          elsif scan RE::PHP_END
-            kind = :inline_delimiter
+          elsif match = scan(RE::PHP_END)
+            encoder.text_token match, :inline_delimiter
             states = [:initial]
           
           elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
-            tokens << [:open, :string]
-            warn 'heredoc in heredoc?' if heredoc_delimiter
+            encoder.begin_group :string
+            # warn 'heredoc in heredoc?' if heredoc_delimiter
             heredoc_delimiter = Regexp.escape(self[1] || self[2] || self[3])
-            kind = :delimiter
+            encoder.text_token match, :delimiter
             states.push self[3] ? :sqstring : :dqstring
             heredoc_delimiter = /#{heredoc_delimiter}(?=;?$)/
           
@@ -379,152 +366,141 @@ module Scanners
               label_expected = true if match == ':'
               case_expected = false
             end
-            kind = :operator
+            encoder.text_token match, :operator
           
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
           
           end
         
         when :sqstring
-          if scan(heredoc_delimiter ? /[^\\\n]+/ : /[^'\\]+/)
-            kind = :content
-          elsif !heredoc_delimiter && scan(/'/)
-            tokens << [matched, :delimiter]
-            tokens << [:close, :string]
+          if match = scan(heredoc_delimiter ? /[^\\\n]+/ : /[^'\\]+/)
+            encoder.text_token match, :content
+          elsif !heredoc_delimiter && match = scan(/'/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
             delimiter = nil
             label_expected = false
             states.pop
-            next
           elsif heredoc_delimiter && match = scan(/\n/)
-            kind = :content
             if scan heredoc_delimiter
-              tokens << ["\n", :content]
-              tokens << [matched, :delimiter]
-              tokens << [:close, :string]
+              encoder.text_token "\n", :content
+              encoder.text_token matched, :delimiter
+              encoder.end_group :string
               heredoc_delimiter = nil
               label_expected = false
               states.pop
-              next
+            else
+              encoder.text_token match, :content
             end
-          elsif scan(heredoc_delimiter ? /\\\\/ : /\\[\\'\n]/)
-            kind = :char
-          elsif scan(/\\./m)
-            kind = :content
-          elsif scan(/\\/)
-            kind = :error
+          elsif match = scan(heredoc_delimiter ? /\\\\/ : /\\[\\'\n]/)
+            encoder.text_token match, :char
+          elsif match = scan(/\\./m)
+            encoder.text_token match, :content
+          elsif match = scan(/\\/)
+            encoder.text_token match, :error
+          else
+            states.pop
           end
         
         when :dqstring
-          if scan(heredoc_delimiter ? /[^${\\\n]+/ : (delimiter == '"' ? /[^"${\\]+/ : /[^`${\\]+/))
-            kind = :content
-          elsif !heredoc_delimiter && scan(delimiter == '"' ? /"/ : /`/)
-            tokens << [matched, :delimiter]
-            tokens << [:close, :string]
+          if match = scan(heredoc_delimiter ? /[^${\\\n]+/ : (delimiter == '"' ? /[^"${\\]+/ : /[^`${\\]+/))
+            encoder.text_token match, :content
+          elsif !heredoc_delimiter && match = scan(delimiter == '"' ? /"/ : /`/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
             delimiter = nil
             label_expected = false
             states.pop
-            next
           elsif heredoc_delimiter && match = scan(/\n/)
-            kind = :content
             if scan heredoc_delimiter
-              tokens << ["\n", :content]
-              tokens << [matched, :delimiter]
-              tokens << [:close, :string]
+              encoder.text_token "\n", :content
+              encoder.text_token matched, :delimiter
+              encoder.end_group :string
               heredoc_delimiter = nil
               label_expected = false
               states.pop
-              next
+            else
+              encoder.text_token match, :content
             end
-          elsif scan(/\\(?:x[0-9A-Fa-f]{1,2}|[0-7]{1,3})/)
-            kind = :char
-          elsif scan(heredoc_delimiter ? /\\[nrtvf\\$]/ : (delimiter == '"' ? /\\[nrtvf\\$"]/ : /\\[nrtvf\\$`]/))
-            kind = :char
-          elsif scan(/\\./m)
-            kind = :content
-          elsif scan(/\\/)
-            kind = :error
+          elsif match = scan(/\\(?:x[0-9A-Fa-f]{1,2}|[0-7]{1,3})/)
+            encoder.text_token match, :char
+          elsif match = scan(heredoc_delimiter ? /\\[nrtvf\\$]/ : (delimiter == '"' ? /\\[nrtvf\\$"]/ : /\\[nrtvf\\$`]/))
+            encoder.text_token match, :char
+          elsif match = scan(/\\./m)
+            encoder.text_token match, :content
+          elsif match = scan(/\\/)
+            encoder.text_token match, :error
           elsif match = scan(/#{RE::VARIABLE}/o)
-            kind = :local_variable
             if check(/\[#{RE::IDENTIFIER}\]/o)
-              tokens << [:open, :inline]
-              tokens << [match, :local_variable]
-              tokens << [scan(/\[/), :operator]
-              tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
-              tokens << [scan(/\]/), :operator]
-              tokens << [:close, :inline]
-              next
+              encoder.begin_group :inline
+              encoder.text_token match, :local_variable
+              encoder.text_token scan(/\[/), :operator
+              encoder.text_token scan(/#{RE::IDENTIFIER}/o), :ident
+              encoder.text_token scan(/\]/), :operator
+              encoder.end_group :inline
             elsif check(/\[/)
               match << scan(/\[['"]?#{RE::IDENTIFIER}?['"]?\]?/o)
-              kind = :error
+              encoder.text_token match, :error
             elsif check(/->#{RE::IDENTIFIER}/o)
-              tokens << [:open, :inline]
-              tokens << [match, :local_variable]
-              tokens << [scan(/->/), :operator]
-              tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
-              tokens << [:close, :inline]
-              next
+              encoder.begin_group :inline
+              encoder.text_token match, :local_variable
+              encoder.text_token scan(/->/), :operator
+              encoder.text_token scan(/#{RE::IDENTIFIER}/o), :ident
+              encoder.end_group :inline
             elsif check(/->/)
               match << scan(/->/)
-              kind = :error
+              encoder.text_token match, :error
+            else
+              encoder.text_token match, :local_variable
             end
           elsif match = scan(/\{/)
             if check(/\$/)
-              kind = :delimiter
+              encoder.begin_group :inline
               states[-1] = [states.last, delimiter]
               delimiter = nil
               states.push :php
-              tokens << [:open, :inline]
+              encoder.text_token match, :delimiter
             else
-              kind = :string
+              encoder.text_token match, :content
             end
-          elsif scan(/\$\{#{RE::IDENTIFIER}\}/o)
-            kind = :local_variable
-          elsif scan(/\$/)
-            kind = :content
+          elsif match = scan(/\$\{#{RE::IDENTIFIER}\}/o)
+            encoder.text_token match, :local_variable
+          elsif match = scan(/\$/)
+            encoder.text_token match, :content
+          else
+            states.pop
           end
         
         when :class_expected
-          if scan(/\s+/)
-            kind = :space
+          if match = scan(/\s+/)
+            encoder.text_token match, :space
           elsif match = scan(/#{RE::IDENTIFIER}/o)
-            kind = :class
+            encoder.text_token match, :class
             states.pop
           else
             states.pop
-            next
           end
         
         when :function_expected
-          if scan(/\s+/)
-            kind = :space
-          elsif scan(/&/)
-            kind = :operator
+          if match = scan(/\s+/)
+            encoder.text_token match, :space
+          elsif match = scan(/&/)
+            encoder.text_token match, :operator
           elsif match = scan(/#{RE::IDENTIFIER}/o)
-            kind = :function
+            encoder.text_token match, :function
             states.pop
           else
             states.pop
-            next
           end
         
         else
-          raise_inspect 'Unknown state!', tokens, states
+          raise_inspect 'Unknown state!', encoder, states
         end
         
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, states
-        end
-        raise_inspect 'Empty token', tokens, states unless match
-        
-        tokens << [match, kind]
-        
       end
       
-      tokens
+      encoder
     end
     
   end

data/lib/coderay/scanners/python.rb

@@ -1,12 +1,12 @@
 module CodeRay
 module Scanners
   
-  # Bases on pygments' PythonLexer, see
+  # Scanner for Python. Supports Python 3.
+  # 
+  # Based on pygments' PythonLexer, see
   # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
   class Python < Scanner
     
-    include Streamable
-    
     register_for :python
     file_extension 'py'
     
@@ -16,11 +16,11 @@ module Scanners
       'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
       'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
       'nonlocal',  # new in Python 3
-    ]
+    ]  # :nodoc:
     
     OLD_KEYWORDS = [
       'exec', 'print',  # gone in Python 3
-    ]
+    ]  # :nodoc:
     
     PREDEFINED_METHODS_AND_TYPES = %w[
       __import__ abs all any apply basestring bin bool buffer
@@ -32,7 +32,7 @@ module Scanners
       raw_input reduce reload repr reversed round set setattr slice
       sorted staticmethod str sum super tuple type unichr unicode
       vars xrange zip
-    ]
+    ]  # :nodoc:
     
     PREDEFINED_EXCEPTIONS = %w[
       ArithmeticError AssertionError AttributeError
@@ -47,23 +47,23 @@ module Scanners
       TypeError UnboundLocalError UnicodeDecodeError
       UnicodeEncodeError UnicodeError UnicodeTranslateError
       UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
-    ]
+    ]  # :nodoc:
     
     PREDEFINED_VARIABLES_AND_CONSTANTS = [
-      'False', 'True', 'None', # "keywords" since Python 3
+      'False', 'True', 'None',  # "keywords" since Python 3
       'self', 'Ellipsis', 'NotImplemented',
-    ]
+    ]  # :nodoc:
     
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(OLD_KEYWORDS, :old_keyword).
       add(PREDEFINED_METHODS_AND_TYPES, :predefined).
-      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
-      add(PREDEFINED_EXCEPTIONS, :exception)
+      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_EXCEPTIONS, :exception)  # :nodoc:
     
-    NAME = / [^\W\d] \w* /x
-    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
+    NAME = / [^\W\d] \w* /x  # :nodoc:
+    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x  # :nodoc:
     
     OPERATOR = /
       \.\.\. |          # ellipsis
@@ -73,95 +73,103 @@ module Scanners
       [-+*\/%&|^]=? |   # ordinary math and binary logic
       [~`] |            # binary complement and inspection
       <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
-    /x
+    /x  # :nodoc:
     
-    STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
-      h[delimiter] = Regexp.union delimiter
-    end
+    STRING_DELIMITER_REGEXP = Hash.new { |h, delimiter|
+      h[delimiter] = Regexp.union delimiter  # :nodoc:
+    }
     
-    STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
-      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
-    end
+    STRING_CONTENT_REGEXP = Hash.new { |h, delimiter|
+      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x  # :nodoc:
+    }
     
     DEF_NEW_STATE = WordList.new(:initial).
       add(%w(def), :def_expected).
       add(%w(import from), :include_expected).
-      add(%w(class), :class_expected)
+      add(%w(class), :class_expected)  # :nodoc:
     
     DESCRIPTOR = /
       #{NAME}
       (?: \. #{NAME} )*
       | \*
-    /x
+    /x  # :nodoc:
+    
+    DOCSTRING_COMING = /
+      [ \t]* u?r? ("""|''')
+    /x  # :nodoc:
     
-    def scan_tokens tokens, options
+  protected
+    
+    def scan_tokens encoder, options
       
       state = :initial
       string_delimiter = nil
       string_raw = false
+      string_type = nil
+      docstring_coming = match?(/#{DOCSTRING_COMING}/o)
       last_token_dot = false
       unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
       from_import_state = []
       
       until eos?
         
-        kind = nil
-        match = nil
-        
         if state == :string
-          if scan(STRING_DELIMITER_REGEXP[string_delimiter])
-            tokens << [matched, :delimiter]
-            tokens << [:close, :string]
+          if match = scan(STRING_DELIMITER_REGEXP[string_delimiter])
+            encoder.text_token match, :delimiter
+            encoder.end_group string_type
+            string_type = nil
             state = :initial
             next
-          elsif string_delimiter.size == 3 && scan(/\n/)
-            kind = :content
-          elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
-            kind = :content
-          elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
-            kind = :char
-          elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
-            kind = :char
-          elsif scan(/ \\ . /x)
-            kind = :content
-          elsif scan(/ \\ | $ /x)
-            tokens << [:close, :string]
-            kind = :error
+          elsif string_delimiter.size == 3 && match = scan(/\n/)
+            encoder.text_token match, :content
+          elsif match = scan(STRING_CONTENT_REGEXP[string_delimiter])
+            encoder.text_token match, :content
+          elsif !string_raw && match = scan(/ \\ #{ESCAPE} /ox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ #{UNICODE_ESCAPE} /ox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ . /x)
+            encoder.text_token match, :content
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group string_type
+            string_type = nil
+            encoder.text_token match, :error
             state = :initial
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state
           end
         
-        elsif match = scan(/ [ \t]+ | \\\n /x)
-          tokens << [match, :space]
-          next
-        
-        elsif match = scan(/\n/)
-          tokens << [match, :space]
-          state = :initial if state == :include_expected
+        elsif match = scan(/ [ \t]+ | \\?\n /x)
+          encoder.text_token match, :space
+          if match == "\n"
+            state = :initial if state == :include_expected
+            docstring_coming = true if match?(/#{DOCSTRING_COMING}/o)
+          end
           next
         
         elsif match = scan(/ \# [^\n]* /mx)
-          tokens << [match, :comment]
+          encoder.text_token match, :comment
           next
         
         elsif state == :initial
           
-          if scan(/#{OPERATOR}/o)
-            kind = :operator
+          if match = scan(/#{OPERATOR}/o)
+            encoder.text_token match, :operator
           
           elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
-            tokens << [:open, :string]
             string_delimiter = self[2]
+            string_type = docstring_coming ? :docstring : :string
+            docstring_coming = false if docstring_coming
+            encoder.begin_group string_type
             string_raw = false
             modifiers = self[1]
             unless modifiers.empty?
               string_raw = !!modifiers.index(?r)
-              tokens << [modifiers, :modifier]
+              encoder.text_token modifiers, :modifier
               match = string_delimiter
             end
             state = :string
-            kind = :delimiter
+            encoder.text_token match, :delimiter
           
           # TODO: backticks
           
@@ -177,43 +185,45 @@ module Scanners
               state = DEF_NEW_STATE[match]
               from_import_state << match.to_sym if state == :include_expected
             end
+            encoder.text_token match, kind
           
-          elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
-            kind = :decorator
+          elsif match = scan(/@[a-zA-Z0-9_.]+[lL]?/)
+            encoder.text_token match, :decorator
           
-          elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
-            kind = :hex
+          elsif match = scan(/0[xX][0-9A-Fa-f]+[lL]?/)
+            encoder.text_token match, :hex
           
-          elsif scan(/0[bB][01]+[lL]?/)
-            kind = :bin
+          elsif match = scan(/0[bB][01]+[lL]?/)
+            encoder.text_token match, :binary
           
           elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
-            kind = :float
             if scan(/[jJ]/)
               match << matched
-              kind = :imaginary
+              encoder.text_token match, :imaginary
+            else
+              encoder.text_token match, :float
             end
           
-          elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
-            kind = :oct
+          elsif match = scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
+            encoder.text_token match, :octal
           
           elsif match = scan(/\d+([lL])?/)
-            kind = :integer
             if self[1] == nil && scan(/[jJ]/)
               match << matched
-              kind = :imaginary
+              encoder.text_token match, :imaginary
+            else
+              encoder.text_token match, :integer
             end
           
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
           
           end
             
         elsif state == :def_expected
           state = :initial
           if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
-            kind = :method
+            encoder.text_token match, :method
           else
             next
           end
@@ -221,33 +231,34 @@ module Scanners
         elsif state == :class_expected
           state = :initial
           if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
-            kind = :class
+            encoder.text_token match, :class
           else
             next
           end
           
         elsif state == :include_expected
           if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
-            kind = :include
             if match == 'as'
-              kind = :keyword
+              encoder.text_token match, :keyword
               from_import_state << :as
             elsif from_import_state.first == :from && match == 'import'
-              kind = :keyword
+              encoder.text_token match, :keyword
               from_import_state << :import
             elsif from_import_state.last == :as
-              # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
-              kind = :ident
+              # encoder.text_token match, match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
+              encoder.text_token match, :ident
               from_import_state.pop
             elsif IDENT_KIND[match] == :keyword
               unscan
               match = nil
               state = :initial
               next
+            else
+              encoder.text_token match, :include
             end
           elsif match = scan(/,/)
             from_import_state.pop if from_import_state.last == :as
-            kind = :operator
+            encoder.text_token match, :operator
           else
             from_import_state = []
             state = :initial
@@ -255,28 +266,19 @@ module Scanners
           end
           
         else
-          raise_inspect 'Unknown state', tokens, state
+          raise_inspect 'Unknown state', encoder, state
           
         end
         
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, state
-        end
-        raise_inspect 'Empty token', tokens, state unless match
-        
         last_token_dot = match == '.'
         
-        tokens << [match, kind]
-        
       end
       
       if state == :string
-        tokens << [:close, :string]
+        encoder.end_group string_type
       end
       
-      tokens
+      encoder
     end
     
   end