coderay-beta 0.9.1

data/lib/coderay/scanners/cpp.rb

@@ -0,0 +1,228 @@
+module CodeRay
+module Scanners
+
+  class CPlusPlus < Scanner
+
+    include Streamable
+    
+    register_for :cpp
+    file_extension 'cpp'
+    title 'C++'
+    
+    # http://www.cppreference.com/wiki/keywords/start
+    RESERVED_WORDS = [
+      'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
+      'case', 'catch', 'class', 'compl', 'const_cast',
+      'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
+      'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
+      'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
+      'sizeof', 'static_cast', 'struct', 'switch', 'template',
+      'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
+      'while', 'xor', 'xor_eq'
+    ]
+
+    PREDEFINED_TYPES = [
+      'bool', 'char', 'double', 'float', 'int', 'long',
+      'short', 'signed', 'unsigned', 'wchar_t', 'string'
+    ]
+    PREDEFINED_CONSTANTS = [
+      'false', 'true',
+      'EOF', 'NULL',
+    ]
+    PREDEFINED_VARIABLES = [
+      'this'
+    ]
+    DIRECTIVES = [
+      'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
+      'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
+      'volatile'
+    ]
+
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
+      add(PREDEFINED_VARIABLES, :local_variable).
+      add(DIRECTIVES, :directive).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
+
+    ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+    def scan_tokens tokens, options
+
+      state = :initial
+      label_expected = true
+      case_expected = false
+      label_expected_before_preproc_line = nil
+      in_preproc_line = false
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            if in_preproc_line && match != "\\\n" && match.index(?\n)
+              in_preproc_line = false
+              label_expected = label_expected_before_preproc_line
+            end
+            tokens << [match, :space]
+            next
+
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            kind = :comment
+
+          elsif match = scan(/ \# \s* if \s* 0 /x)
+            match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+            kind = :comment
+
+          elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
+            label_expected = match =~ /[;\{\}]/
+            if case_expected
+              label_expected = true if match == ':'
+              case_expected = false
+            end
+            kind = :operator
+
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = IDENT_KIND[match]
+            if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
+              kind = :label
+              match << matched
+            else
+              label_expected = false
+              if kind == :reserved
+                case match
+                when 'class'
+                  state = :class_name_expected
+                when 'case', 'default'
+                  case_expected = true
+                end
+              end
+            end
+
+          elsif scan(/\$/)
+            kind = :ident
+          
+          elsif match = scan(/L?"/)
+            tokens << [:open, :string]
+            if match[0] == ?L
+              tokens << ['L', :modifier]
+              match = '"'
+            end
+            state = :string
+            kind = :delimiter
+
+          elsif scan(/#[ \t]*(\w*)/)
+            kind = :preprocessor
+            in_preproc_line = true
+            label_expected_before_preproc_line = label_expected
+            state = :include_expected if self[1] == 'include'
+
+          elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+            label_expected = false
+            kind = :char
+
+          elsif scan(/0[xX][0-9A-Fa-f]+/)
+            label_expected = false
+            kind = :hex
+
+          elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+            label_expected = false
+            kind = :oct
+
+          elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
+            label_expected = false
+            kind = :integer
+
+          elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+            label_expected = false
+            kind = :float
+
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string
+          if scan(/[^\\"]+/)
+            kind = :content
+          elsif scan(/"/)
+            tokens << ['"', :delimiter]
+            tokens << [:close, :string]
+            state = :initial
+            label_expected = false
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :string]
+            kind = :error
+            state = :initial
+            label_expected = false
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        when :include_expected
+          if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            kind = :include
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            kind = :space
+            state = :initial if match.index ?\n
+
+          else
+            state = :initial
+            next
+
+          end
+        
+        when :class_name_expected
+          if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = :class
+            state = :initial
+
+          elsif match = scan(/\s+/)
+            kind = :space
+
+          else
+            getch
+            kind = :error
+            state = :initial
+
+          end
+          
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        tokens << [match, kind]
+
+      end
+
+      if state == :string
+        tokens << [:close, :string]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/css.rb

@@ -0,0 +1,210 @@
+module CodeRay
+module Scanners
+
+  class CSS < Scanner
+
+    register_for :css
+
+    KINDS_NOT_LOC = [
+      :comment,
+      :class, :pseudo_class, :type,
+      :constant, :directive,
+      :key, :value, :operator, :color, :float,
+      :error, :important,
+    ]
+    
+    module RE
+      NonASCII = /[\x80-\xFF]/
+      Hex = /[0-9a-fA-F]/
+      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
+      NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
+      NL = /\r\n|\r|\n|\f/
+      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
+      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp
+      String = /#{String1}|#{String2}/
+
+      HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
+      Color = /#{HexColor}/
+
+      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Name = /#{NMChar}+/
+      Ident = /-?#{NMStart}#{NMChar}*/
+      AtKeyword = /@#{Ident}/
+      Percentage = /#{Num}%/
+
+      reldimensions = %w[em ex px]
+      absdimensions = %w[in cm mm pt pc]
+      Unit = Regexp.union(*(reldimensions + absdimensions))
+
+      Dimension = /#{Num}#{Unit}/
+
+      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
+      Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
+
+      Id = /##{Name}/
+      Class = /\.#{Name}/
+      PseudoClass = /:#{Name}/
+      AttributeSelector = /\[[^\]]*\]?/
+
+    end
+
+    def scan_tokens tokens, options
+      
+      value_expected = nil
+      states = [:initial]
+
+      until eos?
+
+        kind = nil
+        match = nil
+
+        if scan(/\s+/)
+          kind = :space
+
+        elsif case states.last
+          when :initial, :media
+            if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
+              kind = :type
+            elsif scan RE::Class
+              kind = :class
+            elsif scan RE::Id
+              kind = :constant
+            elsif scan RE::PseudoClass
+              kind = :pseudo_class
+            elsif match = scan(RE::AttributeSelector)
+              # TODO: Improve highlighting inside of attribute selectors.
+              tokens << [:open, :string]
+              tokens << [match[0,1], :delimiter]
+              tokens << [match[1..-2], :content] if match.size > 2
+              tokens << [match[-1,1], :delimiter] if match[-1] == ?]
+              tokens << [:close, :string]
+              next
+            elsif match = scan(/@media/)
+              kind = :directive
+              states.push :media_before_name
+            end
+          
+          when :block
+            if scan(/(?>#{RE::Ident})(?!\()/ox)
+              if value_expected
+                kind = :value
+              else
+                kind = :key
+              end
+            end
+
+          when :media_before_name
+            if scan RE::Ident
+              kind = :type
+              states[-1] = :media_after_name
+            end
+          
+          when :media_after_name
+            if scan(/\{/)
+              kind = :operator
+              states[-1] = :media
+            end
+          
+          when :comment
+            if scan(/(?:[^*\s]|\*(?!\/))+/)
+              kind = :comment
+            elsif scan(/\*\//)
+              kind = :comment
+              states.pop
+            elsif scan(/\s+/)
+              kind = :space
+            end
+
+          else
+            raise_inspect 'Unknown state', tokens
+
+          end
+
+        elsif scan(/\/\*/)
+          kind = :comment
+          states.push :comment
+
+        elsif scan(/\{/)
+          value_expected = false
+          kind = :operator
+          states.push :block
+
+        elsif scan(/\}/)
+          value_expected = false
+          if states.last == :block || states.last == :media
+            kind = :operator
+            states.pop
+          else
+            kind = :error
+          end
+
+        elsif match = scan(/#{RE::String}/o)
+          tokens << [:open, :string]
+          tokens << [match[0, 1], :delimiter]
+          tokens << [match[1..-2], :content] if match.size > 2
+          tokens << [match[-1, 1], :delimiter] if match.size >= 2
+          tokens << [:close, :string]
+          next
+
+        elsif match = scan(/#{RE::Function}/o)
+          tokens << [:open, :string]
+          start = match[/^\w+\(/]
+          tokens << [start, :delimiter]
+          if match[-1] == ?)
+            tokens << [match[start.size..-2], :content]
+            tokens << [')', :delimiter]
+          else
+            tokens << [match[start.size..-1], :content]
+          end
+          tokens << [:close, :string]
+          next
+
+        elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          kind = :float
+
+        elsif scan(/#{RE::Color}/o)
+          kind = :color
+
+        elsif scan(/! *important/)
+          kind = :important
+
+        elsif scan(/rgb\([^()\n]*\)?/)
+          kind = :color
+
+        elsif scan(/#{RE::AtKeyword}/o)
+          kind = :directive
+
+        elsif match = scan(/ [+>:;,.=()\/] /x)
+          if match == ':'
+            value_expected = true
+          elsif match == ';'
+            value_expected = false
+          end
+          kind = :operator
+
+        else
+          getch
+          kind = :error
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        tokens << [match, kind]
+
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/debug.rb

@@ -0,0 +1,62 @@
+module CodeRay
+module Scanners
+
+  # = Debug Scanner
+  class Debug < Scanner
+
+    include Streamable
+    register_for :debug
+    file_extension 'raydebug'
+    title 'CodeRay Token Dump'
+
+  protected
+    def scan_tokens tokens, options
+
+      opened_tokens = []
+
+      until eos?
+
+        kind = nil
+        match = nil
+
+          if scan(/\s+/)
+            tokens << [matched, :space]
+            next
+            
+          elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
+            kind = self[1].to_sym
+            match = self[2].gsub(/\\(.)/, '\1')
+            
+          elsif scan(/ (\w+) < /x)
+            kind = self[1].to_sym
+            opened_tokens << kind
+            match = :open
+            
+          elsif scan(/ > /x)
+            kind = opened_tokens.pop
+            match = :close
+            
+          else
+            kind = :error
+            getch
+
+          end
+                  
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        tokens << [match, kind]
+        
+      end
+      
+      tokens
+    end
+
+  end
+
+end
+end