coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/cpp.rb

@@ -1,16 +1,17 @@
 module CodeRay
 module Scanners
 
+  # Scanner for C++.
+  # 
+  # Aliases: +cplusplus+, c++
   class CPlusPlus < Scanner
 
-    include Streamable
-    
     register_for :cpp
     file_extension 'cpp'
     title 'C++'
     
-    # http://www.cppreference.com/wiki/keywords/start
-    RESERVED_WORDS = [
+    #-- http://www.cppreference.com/wiki/keywords/start
+    KEYWORDS = [
       'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
       'case', 'catch', 'class', 'compl', 'const_cast',
       'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
@@ -18,37 +19,39 @@ module Scanners
       'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
       'sizeof', 'static_cast', 'struct', 'switch', 'template',
       'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
-      'while', 'xor', 'xor_eq'
-    ]
-
+      'while', 'xor', 'xor_eq',
+    ]  # :nodoc:
+    
     PREDEFINED_TYPES = [
       'bool', 'char', 'double', 'float', 'int', 'long',
-      'short', 'signed', 'unsigned', 'wchar_t', 'string'
-    ]
+      'short', 'signed', 'unsigned', 'wchar_t', 'string',
+    ]  # :nodoc:
     PREDEFINED_CONSTANTS = [
       'false', 'true',
       'EOF', 'NULL',
-    ]
+    ]  # :nodoc:
     PREDEFINED_VARIABLES = [
-      'this'
-    ]
+      'this',
+    ]  # :nodoc:
     DIRECTIVES = [
       'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
       'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
-      'volatile'
-    ]
-
+      'volatile',
+    ]  # :nodoc:
+    
     IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_TYPES, :pre_type).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
       add(PREDEFINED_VARIABLES, :local_variable).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
-
-    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
 
-    def scan_tokens tokens, options
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    
+  protected
+    
+    def scan_tokens encoder, options
 
       state = :initial
       label_expected = true
@@ -58,9 +61,6 @@ module Scanners
 
       until eos?
 
-        kind = nil
-        match = nil
-        
         case state
 
         when :initial
@@ -70,15 +70,14 @@ module Scanners
               in_preproc_line = false
               label_expected = label_expected_before_preproc_line
             end
-            tokens << [match, :space]
-            next
+            encoder.text_token match, :space
 
-          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
-            kind = :comment
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            encoder.text_token match, :comment
 
           elsif match = scan(/ \# \s* if \s* 0 /x)
             match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
-            kind = :comment
+            encoder.text_token match, :comment
 
           elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
             label_expected = match =~ /[;\{\}]/
@@ -86,7 +85,7 @@ module Scanners
               label_expected = true if match == ':'
               case_expected = false
             end
-            kind = :operator
+            encoder.text_token match, :operator
 
           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
             kind = IDENT_KIND[match]
@@ -95,7 +94,7 @@ module Scanners
               match << matched
             else
               label_expected = false
-              if kind == :reserved
+              if kind == :keyword
                 case match
                 when 'class'
                   state = :class_name_expected
@@ -104,122 +103,110 @@ module Scanners
                 end
               end
             end
+            encoder.text_token match, kind
 
-          elsif scan(/\$/)
-            kind = :ident
+          elsif match = scan(/\$/)
+            encoder.text_token match, :ident
           
           elsif match = scan(/L?"/)
-            tokens << [:open, :string]
+            encoder.begin_group :string
             if match[0] == ?L
-              tokens << ['L', :modifier]
+              encoder.text_token match, 'L', :modifier
               match = '"'
             end
             state = :string
-            kind = :delimiter
+            encoder.text_token match, :delimiter
 
-          elsif scan(/#[ \t]*(\w*)/)
-            kind = :preprocessor
+          elsif match = scan(/#[ \t]*(\w*)/)
+            encoder.text_token match, :preprocessor
             in_preproc_line = true
             label_expected_before_preproc_line = label_expected
             state = :include_expected if self[1] == 'include'
 
-          elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+          elsif match = scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
             label_expected = false
-            kind = :char
+            encoder.text_token match, :char
 
-          elsif scan(/0[xX][0-9A-Fa-f]+/)
+          elsif match = scan(/0[xX][0-9A-Fa-f]+/)
             label_expected = false
-            kind = :hex
+            encoder.text_token match, :hex
 
-          elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+          elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
             label_expected = false
-            kind = :oct
+            encoder.text_token match, :octal
 
-          elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
+          elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
             label_expected = false
-            kind = :integer
+            encoder.text_token match, :integer
 
-          elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+          elsif match = scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
             label_expected = false
-            kind = :float
+            encoder.text_token match, :float
 
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
 
           end
 
         when :string
-          if scan(/[^\\"]+/)
-            kind = :content
-          elsif scan(/"/)
-            tokens << ['"', :delimiter]
-            tokens << [:close, :string]
+          if match = scan(/[^\\"]+/)
+            encoder.text_token match, :content
+          elsif match = scan(/"/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
             state = :initial
             label_expected = false
-            next
-          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            kind = :char
-          elsif scan(/ \\ | $ /x)
-            tokens << [:close, :string]
-            kind = :error
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group :string
+            encoder.text_token match, :error
             state = :initial
             label_expected = false
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
 
         when :include_expected
-          if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
-            kind = :include
+          if match = scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            encoder.text_token match, :include
             state = :initial
 
           elsif match = scan(/\s+/)
-            kind = :space
+            encoder.text_token match, :space
             state = :initial if match.index ?\n
 
           else
             state = :initial
-            next
 
           end
         
         when :class_name_expected
-          if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
-            kind = :class
+          if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            encoder.text_token match, :class
             state = :initial
 
           elsif match = scan(/\s+/)
-            kind = :space
+            encoder.text_token match, :space
 
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
             state = :initial
 
           end
           
         else
-          raise_inspect 'Unknown state', tokens
+          raise_inspect 'Unknown state', encoder
 
         end
 
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
-        end
-        raise_inspect 'Empty token', tokens unless match
-
-        tokens << [match, kind]
-
       end
 
       if state == :string
-        tokens << [:close, :string]
+        encoder.end_group :string
       end
 
-      tokens
+      encoder
     end
 
   end

data/lib/coderay/scanners/css.rb

@@ -2,208 +2,191 @@ module CodeRay
 module Scanners
 
   class CSS < Scanner
-
+    
     register_for :css
-
+    
     KINDS_NOT_LOC = [
       :comment,
       :class, :pseudo_class, :type,
       :constant, :directive,
-      :key, :value, :operator, :color, :float, :content, :delimiter,
+      :key, :value, :operator, :color, :float, :string,
       :error, :important,
-    ]
+    ]  # :nodoc:
     
-    module RE
+    module RE  # :nodoc:
       Hex = /[0-9a-fA-F]/
       Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
       Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
       NMChar = /[-_a-zA-Z0-9]|#{Escape}/
       NMStart = /[_a-zA-Z]|#{Escape}/
       NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp
+      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
       String = /#{String1}|#{String2}/
-
+      
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       Color = /#{HexColor}/
-
+      
       Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
       Percentage = /#{Num}%/
-
+      
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
       Unit = Regexp.union(*(reldimensions + absdimensions))
-
+      
       Dimension = /#{Num}#{Unit}/
-
+      
       Comment = %r! /\* (?: .*? \*/ | .* ) !mx
-      Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
-
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
+      
       Id = /##{Name}/
       Class = /\.#{Name}/
       PseudoClass = /:#{Name}/
       AttributeSelector = /\[[^\]]*\]?/
-
     end
-
-    def scan_tokens tokens, options
+    
+  protected
+    
+    def scan_tokens encoder, options
       
       value_expected = nil
       states = [:initial]
-
+      
       until eos?
-
-        kind = nil
-        match = nil
-
-        if scan(/\s+/)
-          kind = :space
-
+        
+        if match = scan(/\s+/)
+          encoder.text_token match, :space
+          
         elsif case states.last
           when :initial, :media
-            if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              kind = :type
-            elsif scan RE::Class
-              kind = :class
-            elsif scan RE::Id
-              kind = :constant
-            elsif scan RE::PseudoClass
-              kind = :pseudo_class
+            if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
+              encoder.text_token match, :type
+              next
+            elsif match = scan(RE::Class)
+              encoder.text_token match, :class
+              next
+            elsif match = scan(RE::Id)
+              encoder.text_token match, :constant
+              next
+            elsif match = scan(RE::PseudoClass)
+              encoder.text_token match, :pseudo_class
+              next
             elsif match = scan(RE::AttributeSelector)
               # TODO: Improve highlighting inside of attribute selectors.
-              tokens << [:open, :string]
-              tokens << [match[0,1], :delimiter]
-              tokens << [match[1..-2], :content] if match.size > 2
-              tokens << [match[-1,1], :delimiter] if match[-1] == ?]
-              tokens << [:close, :string]
+              encoder.text_token match[0,1], :operator
+              encoder.text_token match[1..-2], :attribute_name if match.size > 2
+              encoder.text_token match[-1,1], :operator if match[-1] == ?]
               next
             elsif match = scan(/@media/)
-              kind = :directive
+              encoder.text_token match, :directive
               states.push :media_before_name
+              next
             end
           
           when :block
-            if scan(/(?>#{RE::Ident})(?!\()/ox)
+            if match = scan(/(?>#{RE::Ident})(?!\()/ox)
               if value_expected
-                kind = :value
+                encoder.text_token match, :value
               else
-                kind = :key
+                encoder.text_token match, :key
               end
+              next
             end
-
+            
           when :media_before_name
-            if scan RE::Ident
-              kind = :type
+            if match = scan(RE::Ident)
+              encoder.text_token match, :type
               states[-1] = :media_after_name
+              next
             end
           
           when :media_after_name
-            if scan(/\{/)
-              kind = :operator
+            if match = scan(/\{/)
+              encoder.text_token match, :operator
               states[-1] = :media
+              next
             end
           
-          when :comment
-            if scan(/(?:[^*\s]|\*(?!\/))+/)
-              kind = :comment
-            elsif scan(/\*\//)
-              kind = :comment
-              states.pop
-            elsif scan(/\s+/)
-              kind = :space
-            end
-
           else
-            raise_inspect 'Unknown state', tokens
-
+            #:nocov:
+            raise_inspect 'Unknown state', encoder
+            #:nocov:
+            
           end
-
-        elsif scan(/\/\*/)
-          kind = :comment
-          states.push :comment
-
-        elsif scan(/\{/)
+          
+        elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
+          encoder.text_token match, :comment
+          
+        elsif match = scan(/\{/)
           value_expected = false
-          kind = :operator
+          encoder.text_token match, :operator
           states.push :block
-
-        elsif scan(/\}/)
+          
+        elsif match = scan(/\}/)
           value_expected = false
           if states.last == :block || states.last == :media
-            kind = :operator
+            encoder.text_token match, :operator
             states.pop
           else
-            kind = :error
+            encoder.text_token match, :error
           end
-
+          
         elsif match = scan(/#{RE::String}/o)
-          tokens << [:open, :string]
-          tokens << [match[0, 1], :delimiter]
-          tokens << [match[1..-2], :content] if match.size > 2
-          tokens << [match[-1, 1], :delimiter] if match.size >= 2
-          tokens << [:close, :string]
-          next
-
+          encoder.begin_group :string
+          encoder.text_token match[0, 1], :delimiter
+          encoder.text_token match[1..-2], :content if match.size > 2
+          encoder.text_token match[-1, 1], :delimiter if match.size >= 2
+          encoder.end_group :string
+          
         elsif match = scan(/#{RE::Function}/o)
-          tokens << [:open, :string]
+          encoder.begin_group :string
           start = match[/^\w+\(/]
-          tokens << [start, :delimiter]
+          encoder.text_token start, :delimiter
           if match[-1] == ?)
-            tokens << [match[start.size..-2], :content]
-            tokens << [')', :delimiter]
+            encoder.text_token match[start.size..-2], :content
+            encoder.text_token ')', :delimiter
           else
-            tokens << [match[start.size..-1], :content]
+            encoder.text_token match[start.size..-1], :content
           end
-          tokens << [:close, :string]
-          next
-
-        elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
-          kind = :float
-
-        elsif scan(/#{RE::Color}/o)
-          kind = :color
-
-        elsif scan(/! *important/)
-          kind = :important
-
-        elsif scan(/rgb\([^()\n]*\)?/)
-          kind = :color
-
-        elsif scan(/#{RE::AtKeyword}/o)
-          kind = :directive
-
+          encoder.end_group :string
+          
+        elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          encoder.text_token match, :float
+          
+        elsif match = scan(/#{RE::Color}/o)
+          encoder.text_token match, :color
+          
+        elsif match = scan(/! *important/)
+          encoder.text_token match, :important
+          
+        elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
+          encoder.text_token match, :color
+          
+        elsif match = scan(RE::AtKeyword)
+          encoder.text_token match, :directive
+          
         elsif match = scan(/ [+>:;,.=()\/] /x)
           if match == ':'
             value_expected = true
           elsif match == ';'
             value_expected = false
           end
-          kind = :operator
-
+          encoder.text_token match, :operator
+          
         else
-          getch
-          kind = :error
-
-        end
-
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+          encoder.text_token getch, :error
+          
         end
-        raise_inspect 'Empty token', tokens unless match
-
-        tokens << [match, kind]
-
+        
       end
-
-      tokens
+      
+      encoder
     end
-
+    
   end
-
+  
 end
 end