coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/raydebug.rb

@@ -0,0 +1,66 @@
+module CodeRay
+module Scanners
+
+  # = Debug Scanner
+  # 
+  # Parses the output of the Encoders::Debug encoder.
+  class Raydebug < Scanner
+
+    register_for :raydebug
+    file_extension 'raydebug'
+    title 'CodeRay Token Dump'
+    
+  protected
+    
+    def scan_tokens encoder, options
+
+      opened_tokens = []
+
+      until eos?
+
+        if match = scan(/\s+/)
+          encoder.text_token match, :space
+          
+        elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) /x)
+          kind = self[1]
+          encoder.text_token kind, :class
+          encoder.text_token '(', :operator
+          match = self[2]
+          encoder.text_token match, kind.to_sym
+          encoder.text_token match, :operator if match = scan(/\)/)
+          
+        elsif match = scan(/ (\w+) ([<\[]) /x)
+          kind = self[1]
+          case self[2]
+          when '<'
+            encoder.text_token kind, :class
+          when '['
+            encoder.text_token kind, :class
+          else
+            raise 'CodeRay bug: This case should not be reached.'
+          end
+          kind = kind.to_sym
+          opened_tokens << kind
+          encoder.begin_group kind
+          encoder.text_token self[2], :operator
+          
+        elsif !opened_tokens.empty? && match = scan(/ [>\]] /x)
+          encoder.text_token match, :operator
+          encoder.end_group opened_tokens.pop
+          
+        else
+          encoder.text_token getch, :space
+          
+        end
+        
+      end
+      
+      encoder.end_group opened_tokens.pop until opened_tokens.empty?
+      
+      encoder
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/ruby.rb

@@ -1,7 +1,6 @@
-# encoding: utf-8
 module CodeRay
 module Scanners
-
+  
   # This scanner is really complex, since Ruby _is_ a complex language!
   #
   # It tries to highlight 100% of all common code,
@@ -9,310 +8,240 @@ module Scanners
   #
   # It is optimized for HTML highlighting, and is not very useful for
   # parsing or pretty printing.
-  #
-  # For now, I think it's better than the scanners in VIM or Syntax, or
-  # any highlighter I was able to find, except Caleb's RubyLexer.
-  #
-  # I hope it's also better than the rdoc/irb lexer.
   class Ruby < Scanner
-
-    include Streamable
-
+    
     register_for :ruby
     file_extension 'rb'
-
-    helper :patterns
     
-    if not defined? EncodingError
-      EncodingError = Class.new Exception
+    autoload :Patterns,    'coderay/scanners/ruby/patterns'
+    autoload :StringState, 'coderay/scanners/ruby/string_state'
+    
+    def interpreted_string_state
+      StringState.new :string, true, '"'
     end
-
-  private
-    def scan_tokens tokens, options
-      if string.respond_to?(:encoding)
-        unless string.encoding == Encoding::UTF_8
-          self.string = string.encode Encoding::UTF_8,
-            :invalid => :replace, :undef => :replace, :replace => '?'
-        end
-        unicode = false
-      else
-        unicode = exist?(/[^\x00-\x7f]/)
+    
+  protected
+    
+    def setup
+      @state = :initial
+    end
+    
+    def scan_tokens encoder, options
+      state, heredocs = options[:state] || @state
+      heredocs = heredocs.dup if heredocs.is_a?(Array)
+      
+      if state && state.instance_of?(StringState)
+        encoder.begin_group state.type
       end
       
-      last_token_dot = false
-      value_expected = true
-      heredocs = nil
       last_state = nil
-      state = :initial
-      depth = nil
-      inline_block_stack = []
       
+      method_call_expected = false
+      value_expected = true
+      
+      inline_block_stack = nil
+      inline_block_curly_depth = 0
+      
+      if heredocs
+        state = heredocs.shift
+        encoder.begin_group state.type
+        heredocs = nil if heredocs.empty?
+      end
+      
+      # def_object_stack = nil
+      # def_object_paren_depth = 0
       
       patterns = Patterns  # avoid constant lookup
       
+      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+      
       until eos?
-        match = nil
-        kind = nil
-
-        if state.instance_of? patterns::StringState
-# {{{
-          match = scan_until(state.pattern) || scan_rest
-          tokens << [match, :content] unless match.empty?
-          break if eos?
-
-          if state.heredoc and self[1]  # end of heredoc
-            match = getch.to_s
-            match << scan_until(/$/) unless eos?
-            tokens << [match, :delimiter]
-            tokens << [:close, state.type]
-            state = state.next_state
-            next
-          end
-
-          case match = getch
-
-          when state.delim
-            if state.paren
-              state.paren_depth -= 1
-              if state.paren_depth > 0
-                tokens << [match, :nesting_delimiter]
-                next
-              end
-            end
-            tokens << [match, :delimiter]
-            if state.type == :regexp and not eos?
-              modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
-              tokens << [modifiers, :modifier] unless modifiers.empty?
-            end
-            tokens << [:close, state.type]
-            value_expected = false
-            state = state.next_state
-
-          when '\\'
-            if state.interpreted
-              if esc = scan(/ #{patterns::ESCAPE} /ox)
-                tokens << [match + esc, :char]
-              else
-                tokens << [match, :error]
-              end
+        
+        if state.instance_of? ::Symbol
+          
+          if match = scan(/[ \t\f\v]+/)
+            encoder.text_token match, :space
+            
+          elsif match = scan(/\n/)
+            if heredocs
+              unscan  # heredoc scanning needs \n at start
+              state = heredocs.shift
+              encoder.begin_group state.type
+              heredocs = nil if heredocs.empty?
             else
-              case m = getch
-              when state.delim, '\\'
-                tokens << [match + m, :char]
-              when nil
-                tokens << [match, :error]
-              else
-                tokens << [match + m, :content]
-              end
-            end
-
-          when '#'
-            case peek(1)
-            when '{'
-              inline_block_stack << [state, depth, heredocs]
+              state = :initial if state == :undef_comma_expected
+              encoder.text_token match, :space
               value_expected = true
-              state = :initial
-              depth = 1
-              tokens << [:open, :inline]
-              tokens << [match + getch, :inline_delimiter]
-            when '$', '@'
-              tokens << [match, :escape]
-              last_state = state  # scan one token as normal code, then return here
-              state = :initial
-            else
-              raise_inspect 'else-case # reached; #%p not handled' % peek(1), tokens
             end
-
-          when state.paren
-            state.paren_depth += 1
-            tokens << [match, :nesting_delimiter]
-
-          when /#{patterns::REGEXP_SYMBOLS}/ox
-            tokens << [match, :function]
-
-          else
-            raise_inspect 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
-
-          end
-          next
-# }}}
-        else
-# {{{
-          if match = scan(/[ \t\f]+/)
-            kind = :space
-            match << scan(/\s*/) unless eos? || heredocs
-            value_expected = true if match.index(?\n)
-            tokens << [match, kind]
-            next
             
-          elsif match = scan(/\\?\n/)
-            kind = :space
-            if match == "\n"
-              value_expected = true
-              state = :initial if state == :undef_comma_expected
-            end
+          elsif match = scan(bol? ? / \#(!)?.* | #{patterns::RUBYDOC_OR_DATA} /ox : /\#.*/)
+            encoder.text_token match, self[1] ? :doctype : :comment
+            
+          elsif match = scan(/\\\n/)
             if heredocs
               unscan  # heredoc scanning needs \n at start
+              encoder.text_token scan(/\\/), :space
               state = heredocs.shift
-              tokens << [:open, state.type]
+              encoder.begin_group state.type
               heredocs = nil if heredocs.empty?
-              next
             else
-              match << scan(/\s*/) unless eos?
+              encoder.text_token match, :space
             end
-            tokens << [match, kind]
-            next
-          
-          elsif bol? && match = scan(/\#!.*/)
-            tokens << [match, :doctype]
-            next
             
-          elsif match = scan(/\#.*/) or
-            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
-              kind = :comment
-              tokens << [match, kind]
-              next
-
           elsif state == :initial
-
+            
             # IDENTS #
-            if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
+            if !method_call_expected &&
+               match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
                                       /#{patterns::METHOD_NAME}/o)
-              if last_token_dot
-                kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
-              else
-                if value_expected != :expect_colon && scan(/:(?= )/)
-                  tokens << [match, :key]
-                  match = ':'
-                  kind = :operator
-                else
-                  kind = patterns::IDENT_KIND[match]
-                  if kind == :ident
-                    if match[/\A[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
-                      kind = :constant
-                    end
-                  elsif kind == :reserved
-                    state = patterns::DEF_NEW_STATE[match]
-                    value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
-                  end
+              value_expected = false
+              kind = patterns::IDENT_KIND[match]
+              if kind == :ident
+                if match[/\A[A-Z]/] && !(match[/[!?]$/] || match?(/\(/))
+                  kind = :constant
                 end
+              elsif kind == :keyword
+                state = patterns::KEYWORD_NEW_STATE[match]
+                value_expected = true if patterns::KEYWORDS_EXPECTING_VALUE[match]
               end
-              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
-            
-            elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
-              kind = :ident
-              value_expected = :set if check(unicode ? /#{patterns::VALUE_FOLLOWS}/uo :
-                                                       /#{patterns::VALUE_FOLLOWS}/o)
-
-            # OPERATORS #
-            elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
-              if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
-                value_expected = :set
+              value_expected = true if !value_expected && check(/#{patterns::VALUE_FOLLOWS}/o)
+              encoder.text_token match, kind
+              
+            elsif method_call_expected &&
+               match = scan(unicode ? /#{patterns::METHOD_AFTER_DOT}/uo :
+                                      /#{patterns::METHOD_AFTER_DOT}/o)
+              if method_call_expected == '::' && match[/\A[A-Z]/] && !match?(/\(/)
+                encoder.text_token match, :constant
+              else
+                encoder.text_token match, :ident
               end
-              last_token_dot = :set if self[1]
-              kind = :operator
-              unless inline_block_stack.empty?
+              method_call_expected = false
+              value_expected = check(/#{patterns::VALUE_FOLLOWS}/o)
+              
+            # OPERATORS #
+            elsif !method_call_expected && match = scan(/ (\.(?!\.)|::) | (?: \.\.\.? | ==?=? | [,\(\[\{] )() | [\)\]\}] /x)
+              method_call_expected = self[1]
+              value_expected = !method_call_expected && self[2]
+              if inline_block_stack
                 case match
                 when '{'
-                  depth += 1
+                  inline_block_curly_depth += 1
                 when '}'
-                  depth -= 1
-                  if depth == 0  # closing brace of inline block reached
-                    state, depth, heredocs = inline_block_stack.pop
+                  inline_block_curly_depth -= 1
+                  if inline_block_curly_depth == 0  # closing brace of inline block reached
+                    state, inline_block_curly_depth, heredocs = inline_block_stack.pop
+                    inline_block_stack = nil if inline_block_stack.empty?
                     heredocs = nil if heredocs && heredocs.empty?
-                    tokens << [match, :inline_delimiter]
-                    kind = :inline
-                    match = :close
+                    encoder.text_token match, :inline_delimiter
+                    encoder.end_group :inline
+                    next
                   end
                 end
               end
-
-            elsif match = scan(/ ['"] /mx)
-              tokens << [:open, :string]
-              kind = :delimiter
-              state = patterns::StringState.new :string, match == '"', match  # important for streaming
-
-            elsif match = scan(unicode ? /#{patterns::INSTANCE_VARIABLE}/uo :
-                                         /#{patterns::INSTANCE_VARIABLE}/o)
-              kind = :instance_variable
-
-            elsif value_expected and match = scan(/\//)
-              tokens << [:open, :regexp]
-              kind = :delimiter
-              interpreted = true
-              state = patterns::StringState.new :regexp, interpreted, match
-
-            # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
-            elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
-              kind = self[1] ? :float : :integer
-
+              encoder.text_token match, :operator
+              
             elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
                                          /#{patterns::SYMBOL}/o)
               case delim = match[1]
               when ?', ?"
-                tokens << [:open, :symbol]
-                tokens << [':', :symbol]
+                encoder.begin_group :symbol
+                encoder.text_token ':', :symbol
                 match = delim.chr
-                kind = :delimiter
-                state = patterns::StringState.new :symbol, delim == ?", match
+                encoder.text_token match, :delimiter
+                state = self.class::StringState.new :symbol, delim == ?", match
+              else
+                encoder.text_token match, :symbol
+                value_expected = false
+              end
+              
+            elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx)
+              encoder.begin_group :string
+              if match.size == 1
+                encoder.text_token match, :delimiter
+                state = self.class::StringState.new :string, match == '"', match  # important for streaming
+              else
+                encoder.text_token match[0,1], :delimiter
+                encoder.text_token match[1..-2], :content if match.size > 2
+                encoder.text_token match[-1,1], :delimiter
+                encoder.end_group :string
+                value_expected = false
+              end
+              
+            elsif match = scan(unicode ? /#{patterns::INSTANCE_VARIABLE}/uo :
+                                         /#{patterns::INSTANCE_VARIABLE}/o)
+              value_expected = false
+              encoder.text_token match, :instance_variable
+              
+            elsif value_expected && match = scan(/\//)
+              encoder.begin_group :regexp
+              encoder.text_token match, :delimiter
+              state = self.class::StringState.new :regexp, true, '/'
+              
+            elsif match = scan(value_expected ? /[-+]?#{patterns::NUMERIC}/o : /#{patterns::NUMERIC}/o)
+              if method_call_expected
+                encoder.text_token match, :error
+                method_call_expected = false
               else
-                kind = :symbol
+                encoder.text_token match, self[1] ? :float : :integer  # TODO: send :hex/:octal/:binary
               end
-
-            elsif match = scan(/ -[>=]? | [+!~^]=? | [*|&]{1,2}=? | >>? /x)
-              value_expected = :set
-              kind = :operator
-
-            elsif value_expected and match = scan(unicode ? /#{patterns::HEREDOC_OPEN}/uo :
-                                                            /#{patterns::HEREDOC_OPEN}/o)
-              indented = self[1] == '-'
+              value_expected = false
+              
+            elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x)
+              value_expected = true
+              encoder.text_token match, :operator
+              
+            elsif value_expected && match = scan(/#{patterns::HEREDOC_OPEN}/o)
               quote = self[3]
               delim = self[quote ? 4 : 2]
               kind = patterns::QUOTE_TO_TYPE[quote]
-              tokens << [:open, kind]
-              tokens << [match, :delimiter]
-              match = :close
-              heredoc = patterns::StringState.new kind, quote != '\'', delim, (indented ? :indented : :linestart )
+              encoder.begin_group kind
+              encoder.text_token match, :delimiter
+              encoder.end_group kind
               heredocs ||= []  # create heredocs if empty
-              heredocs << heredoc
-
-            elsif value_expected and match = scan(/#{patterns::FANCY_START_CORRECT}/o)
-              kind, interpreted = *patterns::FancyStringType.fetch(self[1]) do
-                raise_inspect 'Unknown fancy string: %%%p' % k, tokens
-              end
-              tokens << [:open, kind]
-              state = patterns::StringState.new kind, interpreted, self[2]
-              kind = :delimiter
-
-            elsif value_expected and match = scan(unicode ? /#{patterns::CHARACTER}/uo :
-                                                            /#{patterns::CHARACTER}/o)
-              kind = :integer
-
-            elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
-              value_expected = :set
-              kind = :operator
-
+              heredocs << self.class::StringState.new(kind, quote != "'", delim,
+                self[1] == '-' ? :indented : :linestart)
+              value_expected = false
+              
+            elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o)
+              kind = patterns::FANCY_STRING_KIND[self[1]]
+              encoder.begin_group kind
+              state = self.class::StringState.new kind, patterns::FANCY_STRING_INTERPRETED[self[1]], self[2]
+              encoder.text_token match, :delimiter
+              
+            elsif value_expected && match = scan(/#{patterns::CHARACTER}/o)
+              value_expected = false
+              encoder.text_token match, :integer
+              
+            elsif match = scan(/ %=? | <(?:<|=>?)? | \? /x)
+              value_expected = true
+              encoder.text_token match, :operator
+              
             elsif match = scan(/`/)
-              if last_token_dot
-                kind = :operator
-              else
-                tokens << [:open, :shell]
-                kind = :delimiter
-                state = patterns::StringState.new :shell, true, match
-              end
-
+              encoder.begin_group :shell
+              encoder.text_token match, :delimiter
+              state = self.class::StringState.new :shell, true, match
+              
             elsif match = scan(unicode ? /#{patterns::GLOBAL_VARIABLE}/uo :
                                          /#{patterns::GLOBAL_VARIABLE}/o)
-              kind = :global_variable
-
+              encoder.text_token match, :global_variable
+              value_expected = false
+              
             elsif match = scan(unicode ? /#{patterns::CLASS_VARIABLE}/uo :
                                          /#{patterns::CLASS_VARIABLE}/o)
-              kind = :class_variable
-
+              encoder.text_token match, :class_variable
+              value_expected = false
+              
+            elsif match = scan(/\\\z/)
+              encoder.text_token match, :space
+              
             else
-              if !unicode && !string.respond_to?(:encoding)
+              if method_call_expected
+                method_call_expected = false
+                next
+              end
+              unless unicode
                 # check for unicode
-                debug, $DEBUG = $DEBUG, false
+                $DEBUG_BEFORE, $DEBUG = $DEBUG, false
                 begin
                   if check(/./mu).size > 1
                     # seems like we should try again with unicode
@@ -321,124 +250,212 @@ module Scanners
                 rescue
                   # bad unicode char; use getch
                 ensure
-                  $DEBUG = debug
+                  $DEBUG = $DEBUG_BEFORE
                 end
                 next if unicode
               end
-              kind = :error
-              match = scan(unicode ? /./mu : /./m)
-
+              
+              encoder.text_token getch, :error
+              
             end
-
-          elsif state == :def_expected
-            state = :initial
-            if scan(/self\./)
-              tokens << ['self', :pre_constant]
-              tokens << ['.', :operator]
+            
+            if last_state
+              state = last_state
+              last_state = nil
             end
+            
+          elsif state == :def_expected
             if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
                                       /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
-              kind = :method
+              encoder.text_token match, :method
+              state = :initial
+            else
+              last_state = :dot_expected
+              state = :initial
+            end
+            
+          elsif state == :dot_expected
+            if match = scan(/\.|::/)
+              # invalid definition
+              state = :def_expected
+              encoder.text_token match, :operator
             else
-              next
+              state = :initial
             end
-
+            
           elsif state == :module_expected
             if match = scan(/<</)
-              kind = :operator
+              encoder.text_token match, :operator
             else
               state = :initial
-              if match = scan(unicode ? /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/uo :
-                                        /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/o)
-                kind = :class
-              else
-                next
+              if match = scan(unicode ? / (?:#{patterns::IDENT}::)* #{patterns::IDENT} /oux :
+                                        / (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
+                encoder.text_token match, :class
               end
             end
-
+            
           elsif state == :undef_expected
             state = :undef_comma_expected
-            if match = scan(unicode ? /#{patterns::METHOD_NAME_EX}/uo :
-                                      /#{patterns::METHOD_NAME_EX}/o)
-              kind = :method
-            elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
-                                         /#{patterns::SYMBOL}/o)
+            if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
+                                      /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
+              encoder.text_token match, :method
+            elsif match = scan(/#{patterns::SYMBOL}/o)
               case delim = match[1]
               when ?', ?"
-                tokens << [:open, :symbol]
-                tokens << [':', :symbol]
+                encoder.begin_group :symbol
+                encoder.text_token ':', :symbol
                 match = delim.chr
-                kind = :delimiter
-                state = patterns::StringState.new :symbol, delim == ?", match
+                encoder.text_token match, :delimiter
+                state = self.class::StringState.new :symbol, delim == ?", match
                 state.next_state = :undef_comma_expected
               else
-                kind = :symbol
+                encoder.text_token match, :symbol
               end
             else
               state = :initial
-              next
             end
-
-          elsif state == :alias_expected
-            match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
-                                   /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
             
-            if match
-              tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
-              tokens << [self[2], :space]
-              tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
-            end
-            state = :initial
-            next
-
           elsif state == :undef_comma_expected
             if match = scan(/,/)
-              kind = :operator
+              encoder.text_token match, :operator
               state = :undef_expected
             else
               state = :initial
-              next
             end
-
+            
+          elsif state == :alias_expected
+            match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
+                                   /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+            
+            if match
+              encoder.text_token self[1], (self[1][0] == ?: ? :symbol : :method)
+              encoder.text_token self[2], :space
+              encoder.text_token self[3], (self[3][0] == ?: ? :symbol : :method)
+            end
+            state = :initial
+            
+          else
+            #:nocov:
+            raise_inspect 'Unknown state: %p' % [state], encoder
+            #:nocov:
           end
-# }}}
           
-          unless kind == :error
-            if value_expected = value_expected == :set
-              value_expected = :expect_colon if match == '?' || match == 'when'
-            end
-            last_token_dot = last_token_dot == :set
+        else  # StringState
+          
+          match = scan_until(state.pattern) || scan_rest
+          unless match.empty?
+            encoder.text_token match, :content
+            break if eos?
           end
           
-          if $CODERAY_DEBUG and not kind
-            raise_inspect 'Error token %p in line %d' %
-              [[match, kind], line], tokens, state
+          if state.heredoc && self[1]  # end of heredoc
+            match = getch
+            match << scan_until(/$/) unless eos?
+            encoder.text_token match, :delimiter unless match.empty?
+            encoder.end_group state.type
+            state = state.next_state
+            next
           end
-          raise_inspect 'Empty token', tokens unless match
-
-          tokens << [match, kind]
-
-          if last_state
-            state = last_state
-            last_state = nil
+          
+          case match = getch
+          
+          when state.delim
+            if state.paren_depth
+              state.paren_depth -= 1
+              if state.paren_depth > 0
+                encoder.text_token match, :content
+                next
+              end
+            end
+            encoder.text_token match, :delimiter
+            if state.type == :regexp && !eos?
+              match = scan(/#{patterns::REGEXP_MODIFIERS}/o)
+              encoder.text_token match, :modifier unless match.empty?
+            end
+            encoder.end_group state.type
+            value_expected = false
+            state = state.next_state
+            
+          when '\\'
+            if state.interpreted
+              if esc = scan(/#{patterns::ESCAPE}/o)
+                encoder.text_token match + esc, :char
+              else
+                encoder.text_token match, :error
+              end
+            else
+              case esc = getch
+              when nil
+                encoder.text_token match, :content
+              when state.delim, '\\'
+                encoder.text_token match + esc, :char
+              else
+                encoder.text_token match + esc, :content
+              end
+            end
+            
+          when '#'
+            case peek(1)
+            when '{'
+              inline_block_stack ||= []
+              inline_block_stack << [state, inline_block_curly_depth, heredocs]
+              value_expected = true
+              state = :initial
+              inline_block_curly_depth = 1
+              encoder.begin_group :inline
+              encoder.text_token match + getch, :inline_delimiter
+            when '$', '@'
+              encoder.text_token match, :escape
+              last_state = state
+              state = :initial
+            else
+              #:nocov:
+              raise_inspect 'else-case # reached; #%p not handled' % [peek(1)], encoder
+              #:nocov:
+            end
+            
+          when state.opening_paren
+            state.paren_depth += 1
+            encoder.text_token match, :content
+            
+          else
+            #:nocov
+            raise_inspect 'else-case " reached; %p not handled, state = %p' % [match, state], encoder
+            #:nocov:
+            
           end
+          
         end
+        
+      end
+      
+      # cleaning up
+      if state.is_a? StringState
+        encoder.end_group state.type
       end
-
-      inline_block_stack << [state] if state.is_a? patterns::StringState
-      until inline_block_stack.empty?
-        this_block = inline_block_stack.pop
-        tokens << [:close, :inline] if this_block.size > 1
-        state = this_block.first
-        tokens << [:close, state.type]
+      
+      if options[:keep_state]
+        if state.is_a?(StringState) && state.heredoc
+          (heredocs ||= []).unshift state
+          state = :initial
+        elsif heredocs && heredocs.empty?
+          heredocs = nil
+        end
+        @state = state, heredocs
       end
-
-      tokens
+      
+      if inline_block_stack
+        until inline_block_stack.empty?
+          state, = *inline_block_stack.pop
+          encoder.end_group :inline
+          encoder.end_group state.type
+        end
+      end
+      
+      encoder
     end
-
+    
   end
-
+  
 end
 end
-
-# vim:fdm=marker