coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay/scanners/_map.rb

@@ -1,24 +1,22 @@
 module CodeRay
 module Scanners
-  
+
   map \
-    :'c++'       => :cpp,
-    :cplusplus   => :cpp,
-    :ecmascript  => :java_script,
+    :cplusplus => :cpp,
+    :'c++' => :cpp,
+    :ecmascript => :java_script,
     :ecma_script => :java_script,
-    :rhtml       => :erb,
-    :eruby       => :erb,
-    :irb         => :ruby,
-    :javascript  => :java_script,
-    :js          => :java_script,
-    :pascal      => :delphi,
-    :patch       => :diff,
-    :plain       => :text,
-    :plaintext   => :text,
-    :xhtml       => :html,
-    :yml         => :yaml
-  
-  default :text
-  
+    :irb => :ruby,
+    :javascript => :java_script,
+    :js => :java_script,
+    :nitro => :nitro_xhtml,
+    :pascal => :delphi,
+    :patch => :diff,
+    :plain => :plaintext,
+    :xhtml => :html,
+    :yml => :yaml
+
+  default :plain
+
 end
 end

data/lib/coderay/scanners/c.rb

@@ -7,7 +7,7 @@ module Scanners
     register_for :c
     file_extension 'c'
     
-    KEYWORDS = [
+    RESERVED_WORDS = [
       'asm', 'break', 'case', 'continue', 'default', 'do',
       'else', 'enum', 'for', 'goto', 'if', 'return',
       'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
@@ -31,10 +31,10 @@ module Scanners
     ]  # :nodoc:
 
     IDENT_KIND = WordList.new(:ident).
-      add(KEYWORDS, :keyword).
-      add(PREDEFINED_TYPES, :predefined_type).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
+      add(PREDEFINED_CONSTANTS, :pre_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
@@ -65,6 +65,10 @@ module Scanners
           elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
             encoder.text_token match, :comment
 
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            encoder.text_token match, :comment
+
           elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
             label_expected = match =~ /[;\{\}]/
             if case_expected
@@ -80,7 +84,7 @@ module Scanners
               match << matched
             else
               label_expected = false
-              if kind == :keyword
+              if kind == :reserved
                 case match
                 when 'case', 'default'
                   case_expected = true
@@ -89,6 +93,9 @@ module Scanners
             end
             encoder.text_token match, kind
 
+          elsif match = scan(/\$/)
+            encoder.text_token match, :ident
+          
           elsif match = scan(/L?"/)
             encoder.begin_group :string
             if match[0] == ?L
@@ -98,10 +105,6 @@ module Scanners
             encoder.text_token match, :delimiter
             state = :string
 
-          elsif match = scan(/ \# \s* if \s* 0 /x)
-            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
-            encoder.text_token match, :comment
-
           elsif match = scan(/#[ \t]*(\w*)/)
             encoder.text_token match, :preprocessor
             in_preproc_line = true
@@ -112,16 +115,13 @@ module Scanners
             label_expected = false
             encoder.text_token match, :char
 
-          elsif match = scan(/\$/)
-            encoder.text_token match, :ident
-          
           elsif match = scan(/0[xX][0-9A-Fa-f]+/)
             label_expected = false
             encoder.text_token match, :hex
 
           elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
             label_expected = false
-            encoder.text_token match, :octal
+            encoder.text_token match, :oct
 
           elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
             label_expected = false

data/lib/coderay/scanners/cpp.rb

@@ -11,7 +11,7 @@ module Scanners
     title 'C++'
     
     #-- http://www.cppreference.com/wiki/keywords/start
-    KEYWORDS = [
+    RESERVED_WORDS = [
       'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
       'case', 'catch', 'class', 'compl', 'const_cast',
       'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
@@ -40,11 +40,11 @@ module Scanners
     ]  # :nodoc:
     
     IDENT_KIND = WordList.new(:ident).
-      add(KEYWORDS, :keyword).
-      add(PREDEFINED_TYPES, :predefined_type).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
       add(PREDEFINED_VARIABLES, :local_variable).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
+      add(PREDEFINED_CONSTANTS, :pre_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
@@ -94,7 +94,7 @@ module Scanners
               match << matched
             else
               label_expected = false
-              if kind == :keyword
+              if kind == :reserved
                 case match
                 when 'class'
                   state = :class_name_expected
@@ -133,7 +133,7 @@ module Scanners
 
           elsif match = scan(/(?:0[0-7]+)(?![89.eEfF])/)
             label_expected = false
-            encoder.text_token match, :octal
+            encoder.text_token match, :oct
 
           elsif match = scan(/(?:\d+)(?![.eEfF])L?L?/)
             label_expected = false

data/lib/coderay/scanners/css.rb

@@ -2,46 +2,47 @@ module CodeRay
 module Scanners
 
   class CSS < Scanner
-    
+
     register_for :css
-    
+
     KINDS_NOT_LOC = [
       :comment,
       :class, :pseudo_class, :type,
       :constant, :directive,
-      :key, :value, :operator, :color, :float, :string,
+      :key, :value, :operator, :color, :float,
       :error, :important,
     ]  # :nodoc:
     
     module RE  # :nodoc:
+      NonASCII = /[\x80-\xFF]/
       Hex = /[0-9a-fA-F]/
       Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
       Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
-      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
-      NMStart = /[_a-zA-Z]|#{Escape}/
+      NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
+      NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
       NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
+      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
+      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp
       String = /#{String1}|#{String2}/
-      
+
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       Color = /#{HexColor}/
-      
+
       Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
       Percentage = /#{Num}%/
-      
+
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
       Unit = Regexp.union(*(reldimensions + absdimensions))
-      
+
       Dimension = /#{Num}#{Unit}/
-      
+
       Comment = %r! /\* (?: .*? \*/ | .* ) !mx
       Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
-      
+
       Id = /##{Name}/
       Class = /\.#{Name}/
       PseudoClass = /:#{Name}/
@@ -54,36 +55,30 @@ module Scanners
       
       value_expected = nil
       states = [:initial]
-      
+
       until eos?
-        
+
         if match = scan(/\s+/)
           encoder.text_token match, :space
-          
+
         elsif case states.last
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
               encoder.text_token match, :type
-              next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
-              next
             elsif match = scan(RE::Id)
               encoder.text_token match, :constant
-              next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
-              next
             elsif match = scan(RE::AttributeSelector)
               # TODO: Improve highlighting inside of attribute selectors.
               encoder.text_token match[0,1], :operator
               encoder.text_token match[1..-2], :attribute_name if match.size > 2
               encoder.text_token match[-1,1], :operator if match[-1] == ?]
-              next
             elsif match = scan(/@media/)
               encoder.text_token match, :directive
               states.push :media_before_name
-              next
             end
           
           when :block
@@ -93,38 +88,44 @@ module Scanners
               else
                 encoder.text_token match, :key
               end
-              next
             end
-            
+
           when :media_before_name
             if match = scan(RE::Ident)
               encoder.text_token match, :type
               states[-1] = :media_after_name
-              next
             end
           
           when :media_after_name
             if match = scan(/\{/)
               encoder.text_token match, :operator
               states[-1] = :media
-              next
             end
           
+          when :comment
+            if match = scan(/(?:[^*\s]|\*(?!\/))+/)
+              encoder.text_token match, :comment
+            elsif match = scan(/\*\//)
+              encoder.text_token match, :comment
+              states.pop
+            elsif match = scan(/\s+/)
+              encoder.text_token match, :space
+            end
+
           else
-            #:nocov:
             raise_inspect 'Unknown state', encoder
-            #:nocov:
-            
+
           end
-          
-        elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
+
+        elsif match = scan(/\/\*/)
           encoder.text_token match, :comment
-          
+          states.push :comment
+
         elsif match = scan(/\{/)
           value_expected = false
           encoder.text_token match, :operator
           states.push :block
-          
+
         elsif match = scan(/\}/)
           value_expected = false
           if states.last == :block || states.last == :media
@@ -133,14 +134,14 @@ module Scanners
           else
             encoder.text_token match, :error
           end
-          
+
         elsif match = scan(/#{RE::String}/o)
           encoder.begin_group :string
           encoder.text_token match[0, 1], :delimiter
           encoder.text_token match[1..-2], :content if match.size > 2
           encoder.text_token match[-1, 1], :delimiter if match.size >= 2
           encoder.end_group :string
-          
+
         elsif match = scan(/#{RE::Function}/o)
           encoder.begin_group :string
           start = match[/^\w+\(/]
@@ -152,22 +153,22 @@ module Scanners
             encoder.text_token match[start.size..-1], :content
           end
           encoder.end_group :string
-          
+
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
-          
+
         elsif match = scan(/#{RE::Color}/o)
           encoder.text_token match, :color
-          
+
         elsif match = scan(/! *important/)
           encoder.text_token match, :important
-          
+
         elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
           encoder.text_token match, :color
-          
+
         elsif match = scan(RE::AtKeyword)
           encoder.text_token match, :directive
-          
+
         elsif match = scan(/ [+>:;,.=()\/] /x)
           if match == ':'
             value_expected = true
@@ -175,18 +176,18 @@ module Scanners
             value_expected = false
           end
           encoder.text_token match, :operator
-          
+
         else
           encoder.text_token getch, :error
-          
+
         end
-        
+
       end
-      
+
       encoder
     end
-    
+
   end
-  
+
 end
 end

data/lib/coderay/scanners/debug.rb

@@ -1,29 +1,30 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Scanners
-  
+
   # = Debug Scanner
   # 
   # Interprets the output of the Encoders::Debug encoder.
   class Debug < Scanner
-    
+
     register_for :debug
     title 'CodeRay Token Dump Import'
     
   protected
     
     def scan_tokens encoder, options
-      
+
       opened_tokens = []
-      
+
       until eos?
-        
+
         if match = scan(/\s+/)
           encoder.text_token match, :space
           
         elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
           kind = self[1].to_sym
-          match = self[2].gsub(/\\(.)/m, '\1')
-          unless TokenKinds.has_key? kind
+          match = self[2].gsub(/\\(.)/, '\1')
+          unless Tokens::AbbreviationForKind.has_key? kind
             kind = :error
             match = matched
           end
@@ -58,8 +59,53 @@ module Scanners
       
       encoder
     end
-    
+
   end
-  
+
+end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class DebugScannerTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Scanners::Debug < CodeRay::Scanners::Scanner
+    debug = nil
+    assert_nothing_raised do
+      debug = CodeRay.scanner :debug
+    end
+    assert_kind_of CodeRay::Scanners::Scanner, debug
+  end
+  
+  TEST_INPUT = <<-'DEBUG'.chomp
+integer(10)operator((\\\))string<content(test)>test[
+
+  	   
+method([])]
+  DEBUG
+  TEST_OUTPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n\n  \t   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Scanners::Debug.new.tokenize(TEST_INPUT)
+    assert_equal TEST_OUTPUT, CodeRay.scan(TEST_INPUT, :debug)
+  end
+  
 end