coderay 0.8.357 → 0.9.1

data/lib/coderay/scanners/css.rb

@@ -5,6 +5,14 @@ module Scanners
 
     register_for :css
 
+    KINDS_NOT_LOC = [
+      :comment,
+      :class, :pseudo_class, :type,
+      :constant, :directive,
+      :key, :value, :operator, :color, :float,
+      :error, :important,
+    ]
+    
     module RE
       NonASCII = /[\x80-\xFF]/
       Hex = /[0-9a-fA-F]/
@@ -58,7 +66,7 @@ module Scanners
         elsif case states.last
           when :initial, :media
             if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              kind = :keyword
+              kind = :type
             elsif scan RE::Class
               kind = :class
             elsif scan RE::Id

data/lib/coderay/scanners/debug.rb

@@ -7,6 +7,7 @@ module Scanners
     include Streamable
     register_for :debug
     file_extension 'raydebug'
+    title 'CodeRay Token Dump'
 
   protected
     def scan_tokens tokens, options

data/lib/coderay/scanners/delphi.rb

@@ -30,11 +30,11 @@ module Scanners
       'virtual', 'write', 'writeonly'
     ]
 
-    IDENT_KIND = CaseIgnoringWordList.new(:ident, caching=true).
+    IDENT_KIND = CaseIgnoringWordList.new(:ident).
       add(RESERVED_WORDS, :reserved).
       add(DIRECTIVES, :directive)
     
-    NAME_FOLLOWS = CaseIgnoringWordList.new(false, caching=true).
+    NAME_FOLLOWS = CaseIgnoringWordList.new(false).
       add(%w(procedure function .))
 
   private

data/lib/coderay/scanners/diff.rb

@@ -4,6 +4,7 @@ module Scanners
   class Diff < Scanner
     
     register_for :diff
+    title 'diff output'
     
     def scan_tokens tokens, options
       

data/lib/coderay/scanners/groovy.rb

@@ -0,0 +1,263 @@
+module CodeRay
+module Scanners
+
+  load :java
+
+  class Groovy < Java
+
+    include Streamable
+    register_for :groovy
+    
+    # TODO: Check this!
+    GROOVY_KEYWORDS = %w[
+      as assert def in
+    ]
+    KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+      case instanceof new return throw typeof while as assert in
+    ]
+    GROOVY_MAGIC_VARIABLES = %w[ it ]
+    
+    IDENT_KIND = Java::IDENT_KIND.dup.
+      add(GROOVY_KEYWORDS, :keyword).
+      add(GROOVY_MAGIC_VARIABLES, :local_variable)
+    
+    ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # no 4-byte unicode chars? U[a-fA-F0-9]{8}
+    REGEXP_ESCAPE =  / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
+    
+    # TODO: interpretation inside ', ", /
+    STRING_CONTENT_PATTERN = {
+      "'" => /(?>\\[^\\'\n]+|[^\\'\n]+)+/,
+      '"' => /[^\\$"\n]+/,
+      "'''" => /(?>[^\\']+|'(?!''))+/,
+      '"""' => /(?>[^\\$"]+|"(?!""))+/,
+      '/' => /[^\\$\/\n]+/,
+    }
+    
+    def scan_tokens tokens, options
+
+      state = :initial
+      inline_block_stack = []
+      inline_block_paren_depth = nil
+      string_delimiter = nil
+      import_clause = class_name_follows = last_token = after_def = false
+      value_expected = true
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            tokens << [match, :space]
+            if match.index ?\n
+              import_clause = after_def = false
+              value_expected = true unless value_expected
+            end
+            next
+          
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            value_expected = true
+            after_def = false
+            kind = :comment
+          
+          elsif bol? && scan(/ \#!.* /x)
+            kind = :doctype
+          
+          elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
+            after_def = value_expected = false
+            kind = :include
+          
+          elsif match = scan(/ #{IDENT} | \[\] /ox)
+            kind = IDENT_KIND[match]
+            value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+            if last_token == '.'
+              kind = :ident
+            elsif class_name_follows
+              kind = :class
+              class_name_follows = false
+            elsif after_def && check(/\s*[({]/)
+              kind = :method
+              after_def = false
+            elsif kind == :ident && last_token != '?' && check(/:/)
+              kind = :key
+            else
+              class_name_follows = true if match == 'class' || (import_clause && match == 'as')
+              import_clause = match == 'import'
+              after_def = true if match == 'def'
+            end
+          
+          elsif scan(/;/)
+            import_clause = after_def = false
+            value_expected = true
+            kind = :operator
+          
+          elsif scan(/\{/)
+            class_name_follows = after_def = false
+            value_expected = true
+            kind = :operator
+            if !inline_block_stack.empty?
+              inline_block_paren_depth += 1
+            end
+          
+          # TODO: ~'...', ~"..." and ~/.../ style regexps
+          elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
+              && | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
+            value_expected = true
+            value_expected = :regexp if match == '~'
+            after_def = false
+            kind = :operator
+          
+          elsif match = scan(/ [)\]}] /x)
+            value_expected = after_def = false
+            if !inline_block_stack.empty? && match == '}'
+              inline_block_paren_depth -= 1
+              if inline_block_paren_depth == 0  # closing brace of inline block reached
+                tokens << [match, :inline_delimiter]
+                tokens << [:close, :inline]
+                state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
+                next
+              end
+            end
+          
+          elsif check(/[\d.]/)
+            after_def = value_expected = false
+            if scan(/0[xX][0-9A-Fa-f]+/)
+              kind = :hex
+            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              kind = :oct
+            elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+              kind = :float
+            elsif scan(/\d+[lLgG]?/)
+              kind = :integer
+            end
+
+          elsif match = scan(/'''|"""/)
+            after_def = value_expected = false
+            state = :multiline_string
+            tokens << [:open, :string]
+            string_delimiter = match
+            kind = :delimiter
+          
+          # TODO: record.'name'
+          elsif match = scan(/["']/)
+            after_def = value_expected = false
+            state = match == '/' ? :regexp : :string
+            tokens << [:open, state]
+            string_delimiter = match
+            kind = :delimiter
+
+          elsif value_expected && (match = scan(/\//))
+            after_def = value_expected = false
+            tokens << [:open, :regexp]
+            state = :regexp
+            string_delimiter = '/'
+            kind = :delimiter
+
+          elsif scan(/ @ #{IDENT} /ox)
+            after_def = value_expected = false
+            kind = :annotation
+
+          elsif scan(/\//)
+            after_def = false
+            value_expected = true
+            kind = :operator
+          
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string, :regexp, :multiline_string
+          if scan(STRING_CONTENT_PATTERN[string_delimiter])
+            kind = :content
+            
+          elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
+            tokens << [match, :delimiter]
+            if state == :regexp
+              # TODO: regexp modifiers? s, m, x, i?
+              modifiers = scan(/[ix]+/)
+              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+            end
+            state = :string if state == :multiline_string
+            tokens << [:close, state]
+            string_delimiter = nil
+            after_def = value_expected = false
+            state = :initial
+            next
+          
+          elsif (state == :string || state == :multiline_string) &&
+              (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+            if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
+              kind = :content
+            else
+              kind = :char
+            end
+          elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          
+          elsif match = scan(/ \$ #{IDENT} /mox)
+            tokens << [:open, :inline]
+            tokens << ['$', :inline_delimiter]
+            match = match[1..-1]
+            tokens << [match, IDENT_KIND[match]]
+            tokens << [:close, :inline]
+            next
+          elsif match = scan(/ \$ \{ /x)
+            tokens << [:open, :inline]
+            tokens << ['${', :inline_delimiter]
+            inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
+            inline_block_paren_depth = 1
+            state = :initial
+            next
+          
+          elsif scan(/ \$ /mx)
+            kind = :content
+          
+          elsif scan(/ \\. /mx)
+            kind = :content
+          
+          elsif scan(/ \\ | \n /x)
+            tokens << [:close, state]
+            kind = :error
+            after_def = value_expected = false
+            state = :initial
+          
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        last_token = match unless [:space, :comment, :doctype].include? kind
+        
+        tokens << [match, kind]
+
+      end
+
+      if [:multiline_string, :string, :regexp].include? state
+        tokens << [:close, state]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/html.rb

@@ -6,6 +6,13 @@ module Scanners
 
     include Streamable
     register_for :html
+    
+    KINDS_NOT_LOC = [
+      :comment, :doctype, :preprocessor,
+      :tag, :attribute_name, :operator,
+      :attribute_value, :delimiter, :content,
+      :plain, :entity, :error
+    ]
 
     ATTR_NAME = /[\w.:-]+/
     ATTR_VALUE_UNQUOTED = ATTR_NAME
@@ -68,9 +75,9 @@ module Scanners
               kind = :preprocessor
             elsif scan(/<\?.*?\?>|<%.*?%>/m)
               kind = :comment
-            elsif scan(/<\/[-\w_.:]*>/m)
+            elsif scan(/<\/[-\w.:]*>/m)
               kind = :tag
-            elsif match = scan(/<[-\w_.:]+>?/m)
+            elsif match = scan(/<[-\w.:]+>?/m)
               kind = :tag
               state = :attribute unless match[-1] == ?>
             elsif scan(/[^<>&]+/)

data/lib/coderay/scanners/java.rb

@@ -7,30 +7,33 @@ module Scanners
     register_for :java
     helper :builtin_types
     
-    # TODO: Check this!
+    # http://java.sun.com/docs/books/tutorial/java/nutsandbolts/_keywords.html
     KEYWORDS = %w[
-      break case catch continue default do else
-      false finally for if instanceof new null
-      return switch throw true try typeof while
-      debugger export import package
+      assert break case catch continue default do else
+      finally for if instanceof import new package
+      return switch throw try typeof while
+      debugger export
     ]
-    
+    RESERVED = %w[ const goto ]
+    CONSTANTS = %w[ false null true ]
     MAGIC_VARIABLES = %w[ this super ]
     TYPES = %w[
-      boolean byte char class interface double enum float String int long short void
-    ] << '[]'
+      boolean byte char class double enum float int interface long
+      short void
+    ] << '[]'  # because int[] should be highlighted as a type
     DIRECTIVES = %w[
       abstract extends final implements native private protected public
-      static strictfp synchronized threadsafe throws transient volatile
+      static strictfp synchronized throws transient volatile
     ]
     
-    # Reserved for future use.
-    
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
+      add(RESERVED, :reserved).
+      add(CONSTANTS, :pre_constant).
       add(MAGIC_VARIABLES, :local_variable).
       add(TYPES, :type).
       add(BuiltinTypes::List, :pre_type).
+      add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
       add(DIRECTIVES, :directive)
 
     ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
@@ -61,8 +64,9 @@ module Scanners
             tokens << [match, :space]
             next
           
-          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
-            kind = :comment
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            tokens << [match, :comment]
+            next
           
           elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
             kind = :include
@@ -79,7 +83,7 @@ module Scanners
               class_name_follows = true if match == 'class' || match == 'interface'
             end
           
-          elsif scan(/ \.(?!\d) | [,?:(\[)\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
+          elsif scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
             kind = :operator
           
           elsif scan(/;/)

data/lib/coderay/scanners/java_script.rb

@@ -11,14 +11,17 @@ module Scanners
     # The actual JavaScript keywords.
     KEYWORDS = %w[
       break case catch continue default delete do else
-      false finally for function if in instanceof new null
-      return switch throw true try typeof var void while with
+      finally for function if in instanceof new
+      return switch throw try typeof var void while with
+    ]
+    PREDEFINED_CONSTANTS = %w[
+      false null true undefined
     ]
     
     MAGIC_VARIABLES = %w[ this arguments ]  # arguments was introduced in JavaScript 1.4
     
     KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
-      case delete in instanceof new return throw typeof while with
+      case delete in instanceof new return throw typeof with
     ]
     
     # Reserved for future use.
@@ -31,6 +34,7 @@ module Scanners
     
     IDENT_KIND = WordList.new(:ident).
       add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant).
       add(MAGIC_VARIABLES, :local_variable).
       add(KEYWORDS, :keyword)
 
@@ -53,6 +57,7 @@ module Scanners
       string_delimiter = nil
       value_expected = true
       key_expected = false
+      function_expected = false
 
       until eos?
 
@@ -72,7 +77,7 @@ module Scanners
             value_expected = true
             kind = :comment
 
-          elsif check(/\d/)
+          elsif check(/\.?\d/)
             key_expected = value_expected = false
             if scan(/0[xX][0-9A-Fa-f]+/)
               kind = :hex
@@ -83,29 +88,42 @@ module Scanners
             elsif scan(/\d+/)
               kind = :integer
             end
+          
+          elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
+            # FIXME: scan over nested tags
+            xml_scanner.tokenize match
+            value_expected = false
+            next
             
           elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
             value_expected = true
             last_operator = match[-1]
             key_expected = (last_operator == ?{) || (last_operator == ?,)
+            function_expected = false
             kind = :operator
 
           elsif scan(/ [)\]}]+ /x)
-            key_expected = value_expected = false
+            function_expected = key_expected = value_expected = false
             kind = :operator
 
           elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
             kind = IDENT_KIND[match]
             value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+            # TODO: labels
             if kind == :ident
-              if match.index(?$)
+              if match.index(?$)  # $ allowed inside an identifier
                 kind = :predefined
+              elsif function_expected
+                kind = :function
+              elsif check(/\s*[=:]\s*function\b/)
+                kind = :function
               elsif key_expected && check(/\s*:/)
                 kind = :key
               end
             end
+            function_expected = (kind == :keyword) && (match == 'function')
             key_expected = false
-
+          
           elsif match = scan(/["']/)
             if key_expected && check(KEY_CHECK_PATTERN[match])
               state = :key
@@ -158,25 +176,25 @@ module Scanners
           elsif scan(/\\./m)
             kind = :content
           elsif scan(/ \\ | $ /x)
-            tokens << [:close, :delimiter]
+            tokens << [:close, state]
             kind = :error
             key_expected = value_expected = false
             state = :initial
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
           end
 
         else
-          raise_inspect 'Unknown state', tokens, state
+          raise_inspect 'Unknown state', tokens
 
         end
 
         match ||= matched
         if $DEBUG and not kind
           raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, state
+            [[match, kind], line], tokens
         end
-        raise_inspect 'Empty token', tokens, state unless match
+        raise_inspect 'Empty token', tokens unless match
         
         tokens << [match, kind]
 
@@ -189,7 +207,18 @@ module Scanners
       tokens
     end
 
-  end
+  protected
 
+    def reset_instance
+      super
+      @xml_scanner.reset if defined? @xml_scanner
+    end
+
+    def xml_scanner
+      @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+    end
+
+  end
+  
 end
 end