coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/erb.rb

@@ -0,0 +1,81 @@
+module CodeRay
+module Scanners
+  
+  load :html
+  load :ruby
+  
+  # Scanner for HTML ERB templates.
+  class ERB < Scanner
+    
+    register_for :erb
+    title 'HTML ERB Template'
+    
+    KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+    
+    ERB_RUBY_BLOCK = /
+      (<%(?!%)[-=\#]?)
+      ((?>
+        [^\-%]*    # normal*
+        (?>        # special
+          (?: %(?!>) | -(?!%>) )
+          [^\-%]*  # normal*
+        )*
+      ))
+      ((?: -?%> )?)
+    /x  # :nodoc:
+    
+    START_OF_ERB = /
+      <%(?!%)
+    /x  # :nodoc:
+    
+  protected
+    
+    def setup
+      @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+      @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+    end
+    
+    def reset_instance
+      super
+      @html_scanner.reset
+    end
+    
+    def scan_tokens encoder, options
+      
+      until eos?
+        
+        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_rest) and not match.empty?
+          @html_scanner.tokenize match, :tokens => encoder
+          
+        elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
+          start_tag = self[1]
+          code = self[2]
+          end_tag = self[3]
+          
+          encoder.begin_group :inline
+          encoder.text_token start_tag, :inline_delimiter
+          
+          if start_tag == '<%#'
+            encoder.text_token code, :comment
+          else
+            @ruby_scanner.tokenize code, :tokens => encoder
+          end unless code.empty?
+          
+          encoder.text_token end_tag, :inline_delimiter unless end_tag.empty?
+          encoder.end_group :inline
+          
+        else
+          raise_inspect 'else-case reached!', encoder
+          
+        end
+        
+      end
+      
+      encoder
+      
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/groovy.rb

@@ -1,29 +1,29 @@
 module CodeRay
 module Scanners
-
+  
   load :java
-
+  
+  # Scanner for Groovy.
   class Groovy < Java
-
-    include Streamable
+    
     register_for :groovy
     
-    # TODO: Check this!
+    # TODO: check list of keywords
     GROOVY_KEYWORDS = %w[
       as assert def in
-    ]
+    ]  # :nodoc:
     KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
       case instanceof new return throw typeof while as assert in
-    ]
-    GROOVY_MAGIC_VARIABLES = %w[ it ]
+    ]  # :nodoc:
+    GROOVY_MAGIC_VARIABLES = %w[ it ]  # :nodoc:
     
     IDENT_KIND = Java::IDENT_KIND.dup.
       add(GROOVY_KEYWORDS, :keyword).
-      add(GROOVY_MAGIC_VARIABLES, :local_variable)
+      add(GROOVY_MAGIC_VARIABLES, :local_variable)  # :nodoc:
     
-    ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # no 4-byte unicode chars? U[a-fA-F0-9]{8}
-    REGEXP_ESCAPE =  / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
+    ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8}
+    REGEXP_ESCAPE =  / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x  # :nodoc:
     
     # TODO: interpretation inside ', ", /
     STRING_CONTENT_PATTERN = {
@@ -32,45 +32,44 @@ module Scanners
       "'''" => /(?>[^\\']+|'(?!''))+/,
       '"""' => /(?>[^\\$"]+|"(?!""))+/,
       '/' => /[^\\$\/\n]+/,
-    }
+    }  # :nodoc:
+    
+  protected
     
-    def scan_tokens tokens, options
-
+    def scan_tokens encoder, options
+      
       state = :initial
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
       import_clause = class_name_follows = last_token = after_def = false
       value_expected = true
-
+      
       until eos?
-
-        kind = nil
-        match = nil
         
         case state
-
+        
         when :initial
-
+          
           if match = scan(/ \s+ | \\\n /x)
-            tokens << [match, :space]
+            encoder.text_token match, :space
             if match.index ?\n
               import_clause = after_def = false
               value_expected = true unless value_expected
             end
             next
           
-          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
             value_expected = true
             after_def = false
-            kind = :comment
+            encoder.text_token match, :comment
           
-          elsif bol? && scan(/ \#!.* /x)
-            kind = :doctype
+          elsif bol? && match = scan(/ \#!.* /x)
+            encoder.text_token match, :doctype
           
-          elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
+          elsif import_clause && match = scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
             after_def = value_expected = false
-            kind = :include
+            encoder.text_token match, :include
           
           elsif match = scan(/ #{IDENT} | \[\] /ox)
             kind = IDENT_KIND[match]
@@ -90,16 +89,17 @@ module Scanners
               import_clause = match == 'import'
               after_def = true if match == 'def'
             end
+            encoder.text_token match, kind
           
-          elsif scan(/;/)
+          elsif match = scan(/;/)
             import_clause = after_def = false
             value_expected = true
-            kind = :operator
+            encoder.text_token match, :operator
           
-          elsif scan(/\{/)
+          elsif match = scan(/\{/)
             class_name_follows = after_def = false
             value_expected = true
-            kind = :operator
+            encoder.text_token match, :operator
             if !inline_block_stack.empty?
               inline_block_paren_depth += 1
             end
@@ -110,155 +110,146 @@ module Scanners
             value_expected = true
             value_expected = :regexp if match == '~'
             after_def = false
-            kind = :operator
+            encoder.text_token match, :operator
           
           elsif match = scan(/ [)\]}] /x)
             value_expected = after_def = false
             if !inline_block_stack.empty? && match == '}'
               inline_block_paren_depth -= 1
               if inline_block_paren_depth == 0  # closing brace of inline block reached
-                tokens << [match, :inline_delimiter]
-                tokens << [:close, :inline]
+                encoder.text_token match, :inline_delimiter
+                encoder.end_group :inline
                 state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
                 next
               end
             end
-            kind = :operator
+            encoder.text_token match, :operator
           
           elsif check(/[\d.]/)
             after_def = value_expected = false
-            if scan(/0[xX][0-9A-Fa-f]+/)
-              kind = :hex
-            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
-              kind = :oct
-            elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
-              kind = :float
-            elsif scan(/\d+[lLgG]?/)
-              kind = :integer
+            if match = scan(/0[xX][0-9A-Fa-f]+/)
+              encoder.text_token match, :hex
+            elsif match = scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              encoder.text_token match, :octal
+            elsif match = scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+              encoder.text_token match, :float
+            elsif match = scan(/\d+[lLgG]?/)
+              encoder.text_token match, :integer
             end
-
+            
           elsif match = scan(/'''|"""/)
             after_def = value_expected = false
             state = :multiline_string
-            tokens << [:open, :string]
+            encoder.begin_group :string
             string_delimiter = match
-            kind = :delimiter
-          
-          # TODO: record.'name'
+            encoder.text_token match, :delimiter
+            
+          # TODO: record.'name' syntax
           elsif match = scan(/["']/)
             after_def = value_expected = false
             state = match == '/' ? :regexp : :string
-            tokens << [:open, state]
+            encoder.begin_group state
             string_delimiter = match
-            kind = :delimiter
-
-          elsif value_expected && (match = scan(/\//))
+            encoder.text_token match, :delimiter
+            
+          elsif value_expected && match = scan(/\//)
             after_def = value_expected = false
-            tokens << [:open, :regexp]
+            encoder.begin_group :regexp
             state = :regexp
             string_delimiter = '/'
-            kind = :delimiter
-
-          elsif scan(/ @ #{IDENT} /ox)
+            encoder.text_token match, :delimiter
+            
+          elsif match = scan(/ @ #{IDENT} /ox)
             after_def = value_expected = false
-            kind = :annotation
-
-          elsif scan(/\//)
+            encoder.text_token match, :annotation
+            
+          elsif match = scan(/\//)
             after_def = false
             value_expected = true
-            kind = :operator
-          
+            encoder.text_token match, :operator
+            
           else
-            getch
-            kind = :error
-
+            encoder.text_token getch, :error
+            
           end
-
+          
         when :string, :regexp, :multiline_string
-          if scan(STRING_CONTENT_PATTERN[string_delimiter])
-            kind = :content
+          if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+            encoder.text_token match, :content
             
           elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
-            tokens << [match, :delimiter]
+            encoder.text_token match, :delimiter
             if state == :regexp
               # TODO: regexp modifiers? s, m, x, i?
               modifiers = scan(/[ix]+/)
-              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+              encoder.text_token modifiers, :modifier if modifiers && !modifiers.empty?
             end
             state = :string if state == :multiline_string
-            tokens << [:close, state]
+            encoder.end_group state
             string_delimiter = nil
             after_def = value_expected = false
             state = :initial
             next
-          
+            
           elsif (state == :string || state == :multiline_string) &&
               (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
             if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
-              kind = :content
+              encoder.text_token match, :content
             else
-              kind = :char
+              encoder.text_token match, :char
             end
-          elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            kind = :char
-          
+          elsif state == :regexp && match = scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+            
           elsif match = scan(/ \$ #{IDENT} /mox)
-            tokens << [:open, :inline]
-            tokens << ['$', :inline_delimiter]
+            encoder.begin_group :inline
+            encoder.text_token '$', :inline_delimiter
             match = match[1..-1]
-            tokens << [match, IDENT_KIND[match]]
-            tokens << [:close, :inline]
+            encoder.text_token match, IDENT_KIND[match]
+            encoder.end_group :inline
             next
           elsif match = scan(/ \$ \{ /x)
-            tokens << [:open, :inline]
-            tokens << ['${', :inline_delimiter]
+            encoder.begin_group :inline
+            encoder.text_token match, :inline_delimiter
             inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
             inline_block_paren_depth = 1
             state = :initial
             next
-          
-          elsif scan(/ \$ /mx)
-            kind = :content
-          
-          elsif scan(/ \\. /mx)
-            kind = :content
-          
-          elsif scan(/ \\ | \n /x)
-            tokens << [:close, state]
-            kind = :error
+            
+          elsif match = scan(/ \$ /mx)
+            encoder.text_token match, :content
+            
+          elsif match = scan(/ \\. /mx)
+            encoder.text_token match, :content  # TODO: Shouldn't this be :error?
+            
+          elsif match = scan(/ \\ | \n /x)
+            encoder.end_group state
+            encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
-          
+            
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
+            
           end
-
+          
         else
-          raise_inspect 'Unknown state', tokens
-
-        end
-
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+          raise_inspect 'Unknown state', encoder
+          
         end
-        raise_inspect 'Empty token', tokens unless match
         
         last_token = match unless [:space, :comment, :doctype].include? kind
         
-        tokens << [match, kind]
-
       end
-
+      
       if [:multiline_string, :string, :regexp].include? state
-        tokens << [:close, state]
+        encoder.end_group state
       end
-
-      tokens
+      
+      encoder
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/scanners/haml.rb

@@ -0,0 +1,168 @@
+module CodeRay
+module Scanners
+  
+  load :ruby
+  load :html
+  load :java_script
+  
+  class HAML < Scanner
+    
+    register_for :haml
+    title 'HAML Template'
+    
+    KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
+    
+  protected
+    
+    def setup
+      super
+      @ruby_scanner          = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+      @embedded_ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true, :state => @ruby_scanner.interpreted_string_state
+      @html_scanner          = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true
+    end
+    
+    def scan_tokens encoder, options
+      
+      match = nil
+      code = ''
+      
+      until eos?
+        
+        if bol?
+          if match = scan(/!!!.*/)
+            encoder.text_token match, :doctype
+            next
+          end
+          
+          if match = scan(/(?>( *)(\/(?!\[if)|-\#|:javascript|:ruby|:\w+) *)(?=\n)/)
+            encoder.text_token match, :comment
+            
+            code = self[2]
+            if match = scan(/(?:\n+#{self[1]} .*)+/)
+              case code
+              when '/', '-#'
+                encoder.text_token match, :comment
+              when ':javascript'
+                # TODO: recognize #{...} snippets inside JavaScript
+                @java_script_scanner ||= CodeRay.scanner :java_script, :tokens => @tokens, :keep_tokens => true
+                @java_script_scanner.tokenize match, :tokens => encoder
+              when ':ruby'
+                @ruby_scanner.tokenize match, :tokens => encoder
+              when /:\w+/
+                encoder.text_token match, :comment
+              else
+                raise 'else-case reached: %p' % [code]
+              end
+            end
+          end
+          
+          if match = scan(/ +/)
+            encoder.text_token match, :space
+          end
+          
+          if match = scan(/\/.*/)
+            encoder.text_token match, :comment
+            next
+          end
+          
+          if match = scan(/\\/)
+            encoder.text_token match, :plain
+            if match = scan(/.+/)
+              @html_scanner.tokenize match, :tokens => encoder
+            end
+            next
+          end
+          
+          tag = false
+          
+          if match = scan(/%[\w:]+\/?/)
+            encoder.text_token match, :tag
+            # if match = scan(/( +)(.+)/)
+            #   encoder.text_token self[1], :space
+            #   @embedded_ruby_scanner.tokenize self[2], :tokens => encoder
+            # end
+            tag = true
+          end
+          
+          while match = scan(/([.#])[-\w]*\w/)
+            encoder.text_token match, self[1] == '#' ? :constant : :class
+            tag = true
+          end
+          
+          if tag && match = scan(/(\()([^)]+)?(\))?/)
+            # TODO: recognize title=@title, class="widget_#{@widget.number}"
+            encoder.text_token self[1], :plain
+            @html_scanner.tokenize self[2], :tokens => encoder, :state => :attribute if self[2]
+            encoder.text_token self[3], :plain if self[3]
+          end
+          
+          if tag && match = scan(/\{/)
+            encoder.text_token match, :plain
+            
+            code = ''
+            level = 1
+            while true
+              code << scan(/([^\{\},\n]|, *\n?)*/)
+              case match = getch
+              when '{'
+                level += 1
+                code << match
+              when '}'
+                level -= 1
+                if level > 0
+                  code << match
+                else
+                  break
+                end
+              when "\n", ",", nil
+                break
+              end
+            end
+            @ruby_scanner.tokenize code, :tokens => encoder unless code.empty?
+            
+            encoder.text_token match, :plain if match
+          end
+          
+          if tag && match = scan(/(\[)([^\]\n]+)?(\])?/)
+            encoder.text_token self[1], :plain
+            @ruby_scanner.tokenize self[2], :tokens => encoder if self[2]
+            encoder.text_token self[3], :plain if self[3]
+          end
+          
+          if tag && match = scan(/\//)
+            encoder.text_token match, :tag
+          end
+          
+          if scan(/(>?<?[-=]|[&!]=|(& |!)|~)( *)([^,\n\|]+(?:(, *|\|(?=.|\n.*\|$))\n?[^,\n\|]*)*)?/)
+            encoder.text_token self[1] + self[3], :plain
+            if self[4]
+              if self[2]
+                @embedded_ruby_scanner.tokenize self[4], :tokens => encoder
+              else
+                @ruby_scanner.tokenize self[4], :tokens => encoder
+              end
+            end
+          elsif match = scan(/((?:<|><?)(?![!?\/\w]))?(.+)?/)
+            encoder.text_token self[1], :plain if self[1]
+            # TODO: recognize #{...} snippets
+            @html_scanner.tokenize self[2], :tokens => encoder if self[2]
+          end
+          
+        elsif match = scan(/.+/)
+          @html_scanner.tokenize match, :tokens => encoder
+          
+        end
+        
+        if match = scan(/\n/)
+          encoder.text_token match, :space
+        end
+      end
+      
+      encoder
+      
+    end
+    
+  end
+  
+end
+end