coderay 0.7.4.215 → 0.8.260

data/lib/coderay/scanners/_map.rb

@@ -5,8 +5,8 @@ module Scanners
     :plain => :plaintext,
     :pascal => :delphi,
     :irb => :ruby,
-    :xml => :html,
     :xhtml => :nitro_xhtml,
+    :javascript => :java_script,
     :nitro => :nitro_xhtml
 
   default :plain

data/lib/coderay/scanners/c.rb

@@ -4,6 +4,8 @@ module Scanners
   class C < Scanner
 
     register_for :c
+    
+    include Streamable
 
     RESERVED_WORDS = [
       'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
@@ -42,7 +44,7 @@ module Scanners
 
         kind = nil
         match = nil
-
+        
         case state
 
         when :initial

data/lib/coderay/scanners/css.rb

@@ -0,0 +1,181 @@
+module CodeRay
+module Scanners
+
+  class CSS < Scanner
+
+    register_for :css
+
+    module RE
+      NonASCII = /[\x80-\xFF]/
+      Hex = /[0-9a-fA-F]/
+      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
+      NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
+      NL = /\r\n|\r|\n|\f/
+      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
+      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp
+      String = /#{String1}|#{String2}/
+
+      HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
+      Color = /#{HexColor}/
+
+      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Name = /#{NMChar}+/
+      Ident = /-?#{NMStart}#{NMChar}*/
+      AtKeyword = /@#{Ident}/
+      Percentage = /#{Num}%/
+
+      reldimensions = %w[em ex px]
+      absdimensions = %w[in cm mm pt pc]
+      Unit = Regexp.union(*(reldimensions + absdimensions))
+
+      Dimension = /#{Num}#{Unit}/
+
+      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
+      Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
+
+      Id = /##{Name}/
+      Class = /\.#{Name}/
+      PseudoClass = /:#{Name}/
+
+    end
+
+    def scan_tokens tokens, options
+      
+      value_expected = nil
+      states = [:initial]
+
+      until eos?
+
+        kind = nil
+        match = nil
+
+        if scan(/\s+/)
+          kind = :space
+
+        elsif case states.last
+          when :initial
+            if scan(/#{RE::Ident}|\*/ox)
+              kind = :keyword
+            elsif scan RE::Class
+              kind = :class
+            elsif scan RE::Id
+              kind = :constant
+            elsif scan RE::PseudoClass
+              kind = :pseudo_class
+            elsif scan RE::Name
+              kind = :identifier
+            end
+
+          when :block
+            if scan(/(?>#{RE::Ident})(?!\()/ox)
+              if value_expected
+                kind = :value
+              else
+                kind = :key
+              end
+            end
+
+          when :comment
+            if scan(/(?:[^*\s]|\*(?!\/))+/)
+              kind = :comment
+            elsif scan(/\*\//)
+              kind = :comment
+              states.pop
+            elsif scan(/\s+/)
+              kind = :space
+            end
+
+          else
+            raise_inspect 'Unknown state', tokens
+
+          end
+
+        elsif scan(/\/\*/)
+          kind = :comment
+          states.push :comment
+
+        elsif scan(/\{/)
+          value_expected = false
+          kind = :operator
+          states.push :block
+
+        elsif scan(/\}/)
+          value_expected = false
+          if states.last == :block
+            kind = :operator
+            states.pop
+          else
+            kind = :error
+          end
+
+        elsif match = scan(/#{RE::String}/o)
+          tokens << [:open, :string]
+          tokens << [match[0, 1], :delimiter]
+          tokens << [match[1..-2], :content] if match.size > 2
+          tokens << [match[-1, 1], :delimiter] if match.size >= 2
+          tokens << [:close, :string]
+          next
+
+        elsif match = scan(/#{RE::Function}/o)
+          tokens << [:open, :string]
+          start = match[/^\w+\(/]
+          tokens << [start, :delimiter]
+          if match[-1] == ?)
+            tokens << [match[start.size..-2], :content]
+            tokens << [')', :delimiter]
+          else
+            tokens << [match[start.size..-1], :content]
+          end
+          tokens << [:close, :string]
+          next
+
+        elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
+          kind = :float
+
+        elsif scan(/#{RE::Color}/o)
+          kind = :color
+
+        elsif scan(/! *important/)
+          kind = :important
+
+        elsif scan(/rgb\([^()\n]*\)?/)
+          kind = :color
+
+        elsif scan(/#{RE::AtKeyword}/o)
+          kind = :directive
+
+        elsif match = scan(/ [+>:;,.=()\/] /x)
+          if match == ':'
+            value_expected = true
+          elsif match == ';'
+            value_expected = false
+          end
+          kind = :operator
+
+        else
+          getch
+          kind = :error
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        tokens << [match, kind]
+        # tokens << [states.inspect, :error]
+
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/debug.rb

@@ -6,6 +6,7 @@ module Scanners
 
     include Streamable
     register_for :debug
+    file_extension 'raydebug'
 
   protected
     def scan_tokens tokens, options

data/lib/coderay/scanners/delphi.rb

@@ -4,6 +4,7 @@ module Scanners
   class Delphi < Scanner
 
     register_for :delphi
+    file_extension 'pas'
     
     RESERVED_WORDS = [
       'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',

data/lib/coderay/scanners/diff.rb

@@ -0,0 +1,104 @@
+module CodeRay
+module Scanners
+  
+  class Diff < Scanner
+    
+    register_for :diff
+    
+    def scan_tokens tokens, options
+      
+      line_kind = nil
+      state = :initial
+      
+      until eos?
+        kind = match = nil
+        
+        if match = scan(/\n/)
+          if line_kind
+            tokens << [:end_line, line_kind]
+            line_kind = nil
+          end
+          tokens << [match, :space]
+          next
+        end
+        
+        case state
+        
+        when :initial
+          if match = scan(/--- |\+\+\+ |=+|_+/)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/Index: |Property changes on: /)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/Added: /)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+            state = :added
+          elsif match = scan(/\\ /)
+            tokens << [:begin_line, line_kind = :change]
+            tokens << [match, :change]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif scan(/(@@)((?>[^@\n]*))(@@)/)
+            tokens << [:begin_line, line_kind = :change]
+            tokens << [self[1], :change]
+            tokens << [self[2], :plain]
+            tokens << [self[3], :change]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/\+/)
+            tokens << [:begin_line, line_kind = :insert]
+            tokens << [match, :insert]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/-/)
+            tokens << [:begin_line, line_kind = :delete]
+            tokens << [match, :delete]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif scan(/ .*/)
+            kind = :comment
+          elsif scan(/.+/)
+            tokens << [:begin_line, line_kind = :head]
+            kind = :plain
+          else
+            raise_inspect 'else case rached'
+          end
+        
+        when :added
+          if match = scan(/   \+/)
+            tokens << [:begin_line, line_kind = :insert]
+            tokens << [match, :insert]
+            next unless match = scan(/.+/)
+            kind = :plain
+          else
+            state = :initial
+            next
+          end
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+      end
+      
+      tokens << [:end_line, line_kind] if line_kind
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/java.rb

@@ -0,0 +1,179 @@
+module CodeRay
+module Scanners
+
+  class Java < Scanner
+
+    include Streamable
+    register_for :java
+    helper :builtin_types
+    
+    # TODO: Check this!
+    KEYWORDS = %w[
+      break case catch continue default do else
+      false finally for if instanceof new null
+      return switch throw true try typeof while
+      debugger export import package
+    ]
+    
+    MAGIC_VARIABLES = %w[ this super ]
+    TYPES = %w[
+      boolean byte char class interface double enum float String int long short void
+    ] << '[]'
+    DIRECTIVES = %w[
+      abstract extends final implements native private protected public
+      static strictfp synchronized threadsafe throws transient volatile
+    ]
+    
+    # Reserved for future use.
+    
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(MAGIC_VARIABLES, :local_variable).
+      add(TYPES, :type).
+      add(BuiltinTypes::List, :type).
+      add(DIRECTIVES, :directive)
+
+    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+    REGEXP_ESCAPE =  / [bBdDsSwW] /x
+    STRING_CONTENT_PATTERN = {
+      "'" => /[^\\']+/,
+      '"' => /[^\\"]+/,
+      '/' => /[^\\\/]+/,
+    }
+    IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
+    
+    def scan_tokens tokens, options
+
+      state = :initial
+      string_delimiter = nil
+      import_clause = class_name_follows = last_token_dot = false
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            tokens << [match, :space]
+            next
+          
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            kind = :comment
+          
+          elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
+            kind = :include
+          
+          elsif match = scan(/ #{IDENT} | \[\] /ox)
+            kind = IDENT_KIND[match]
+            if last_token_dot
+              kind = :ident
+            elsif class_name_follows
+              kind = :class
+              class_name_follows = false
+            else
+              import_clause = true if match == 'import'
+              class_name_follows = true if match == 'class'
+            end
+          
+          elsif scan(/ \.(?!\d) | [,?:(\[)\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
+            kind = :operator
+          
+          elsif scan(/;/)
+            import_clause = false
+            kind = :operator
+          
+          elsif scan(/\{/)
+            class_name_follows = false
+            kind = :operator
+          
+          elsif check(/[\d.]/)
+            if scan(/0[xX][0-9A-Fa-f]+/)
+              kind = :hex
+            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              kind = :oct
+            elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+              kind = :float
+            elsif scan(/\d+[lL]?/)
+              kind = :integer
+            end
+
+          elsif match = scan(/["']/)
+            tokens << [:open, :string]
+            state = :string
+            string_delimiter = match
+            kind = :delimiter
+
+          elsif scan(/ @ #{IDENT} /ox)
+            kind = :annotation
+
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string, :regexp
+          if scan(STRING_CONTENT_PATTERN[string_delimiter])
+            kind = :content
+          elsif match = scan(/["'\/]/)
+            tokens << [match, :delimiter]
+            if state == :regexp
+              modifiers = scan(/[gim]+/)
+              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+            end
+            tokens << [:close, state]
+            string_delimiter = nil
+            state = :initial
+            next
+          elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+            if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
+              kind = :content
+            else
+              kind = :char
+            end
+          elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/\\./m)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :delimiter]
+            kind = :error
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        last_token_dot = match == '.'
+        
+        tokens << [match, kind]
+
+      end
+
+      if [:string, :regexp].include? state
+        tokens << [:close, state]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end