coderay 1.1.0 → 1.1.3.rc1

data/lib/coderay/scanners.rb

@@ -0,0 +1,27 @@
+require 'strscan'
+
+module CodeRay
+  
+  autoload :WordList, coderay_path('helpers', 'word_list')
+  
+  # = Scanners
+  #
+  # This module holds the Scanner class and its subclasses.
+  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
+  # can be found in coderay/scanners/ruby.
+  #
+  # Scanner also provides methods and constants for the register
+  # mechanism and the [] method that returns the Scanner class
+  # belonging to the given lang.
+  #
+  # See PluginHost.
+  module Scanners
+    
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'scanners'
+    
+    autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner')
+    
+  end
+  
+end

data/lib/coderay/scanners/c.rb

@@ -37,7 +37,7 @@ module Scanners
       add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
     
   protected
     

data/lib/coderay/scanners/cpp.rb

@@ -2,14 +2,14 @@ module CodeRay
 module Scanners
 
   # Scanner for C++.
-  # 
+  #
   # Aliases: +cplusplus+, c++
   class CPlusPlus < Scanner
 
     register_for :cpp
     file_extension 'cpp'
     title 'C++'
-    
+
     #-- http://www.cppreference.com/wiki/keywords/start
     KEYWORDS = [
       'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
@@ -17,28 +17,30 @@ module Scanners
       'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
       'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
       'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
-      'sizeof', 'static_cast', 'struct', 'switch', 'template',
-      'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
+      'sizeof', 'static_assert', 'static_cast', 'struct', 'switch',
+      'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
       'while', 'xor', 'xor_eq',
     ]  # :nodoc:
-    
+
     PREDEFINED_TYPES = [
-      'bool', 'char', 'double', 'float', 'int', 'long',
-      'short', 'signed', 'unsigned', 'wchar_t', 'string',
+      'bool', 'char', 'char16_t', 'char32_t', 'double', 'float',
+      'int', 'long', 'short', 'signed', 'unsigned',
+      'wchar_t', 'string',
     ]  # :nodoc:
     PREDEFINED_CONSTANTS = [
       'false', 'true',
-      'EOF', 'NULL',
+      'EOF', 'NULL', 'nullptr'
     ]  # :nodoc:
     PREDEFINED_VARIABLES = [
       'this',
     ]  # :nodoc:
     DIRECTIVES = [
-      'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
-      'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
-      'volatile',
+      'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit',
+      'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator',
+      'override', 'private', 'protected', 'public', 'register', 'static',
+      'thread_local', 'using', 'virtual', 'void', 'volatile',
     ]  # :nodoc:
-    
+
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(PREDEFINED_TYPES, :predefined_type).
@@ -47,10 +49,10 @@ module Scanners
       add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
-    
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+
   protected
-    
+
     def scan_tokens encoder, options
 
       state = :initial
@@ -107,7 +109,7 @@ module Scanners
 
           elsif match = scan(/\$/)
             encoder.text_token match, :ident
-          
+
           elsif match = scan(/L?"/)
             encoder.begin_group :string
             if match[0] == ?L
@@ -180,7 +182,7 @@ module Scanners
             state = :initial
 
           end
-        
+
         when :class_name_expected
           if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
             encoder.text_token match, :class
@@ -194,7 +196,7 @@ module Scanners
             state = :initial
 
           end
-          
+
         else
           raise_inspect 'Unknown state', encoder
 

data/lib/coderay/scanners/diff.rb

@@ -100,7 +100,7 @@ module Scanners
             next
           elsif match = scan(/-/)
             deleted_lines_count += 1
-            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
+            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
               deleted_lines  = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) }
               inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/)         ; scan(/.*/) }
               
@@ -109,7 +109,7 @@ module Scanners
               for deleted_line, inserted_line in deleted_lines.zip(inserted_lines)
                 pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line
                 content_scanner_entry_state = content_scanner.state
-                deleted_lines_tokenized  << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new)
+                deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new)
                 content_scanner.state = content_scanner_entry_state || :initial
                 inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new)
               end
@@ -212,7 +212,7 @@ module Scanners
       # does not precede the leftmost one from the left.
       j = -1
       j -= 1 while j >= j_min && a[j] == b[j]
-      return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j+1..-1] : ''
+      return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j + 1..-1] : ''
     end
     
   end

data/lib/coderay/scanners/groovy.rb

@@ -22,8 +22,8 @@ module Scanners
       add(GROOVY_MAGIC_VARIABLES, :local_variable)  # :nodoc:
     
     ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8}
-    REGEXP_ESCAPE =  / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x  # :nodoc: no 4-byte unicode chars? U[a-fA-F0-9]{8}
+    REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x  # :nodoc:
     
     # TODO: interpretation inside ', ", /
     STRING_CONTENT_PATTERN = {

data/lib/coderay/scanners/haml.rb

@@ -75,7 +75,7 @@ module Scanners
           
           tag = false
           
-          if match = scan(/%[\w:]+\/?/)
+          if match = scan(/%[-\w:]+\/?/)
             encoder.text_token match, :tag
             # if match = scan(/( +)(.+)/)
             #   encoder.text_token self[1], :space

data/lib/coderay/scanners/java.rb

@@ -20,7 +20,7 @@ module Scanners
     MAGIC_VARIABLES = %w[ this super ]  # :nodoc:
     TYPES = %w[
       boolean byte char class double enum float int interface long
-      short void
+      short void var
     ] << '[]'  # :nodoc: because int[] should be highlighted as a type
     DIRECTIVES = %w[
       abstract extends final implements native private protected public
@@ -36,15 +36,15 @@ module Scanners
       add(BuiltinTypes::List, :predefined_type).
       add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
       add(DIRECTIVES, :directive)  # :nodoc:
-
+    
     ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
     STRING_CONTENT_PATTERN = {
       "'" => /[^\\']+/,
       '"' => /[^\\"]+/,
       '/' => /[^\\\/]+/,
     }  # :nodoc:
-    IDENT = /[a-zA-Z_][A-Za-z_0-9]*/  # :nodoc:
+    IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : Regexp.new('[[[:alpha:]]_][[[:alnum:]]_]*')  # :nodoc:
     
   protected
     

data/lib/coderay/scanners/java_script.rb

@@ -40,8 +40,8 @@ module Scanners
       add(KEYWORDS, :keyword)  # :nodoc:
     
     ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
-    REGEXP_ESCAPE =  / [bBdDsSwW] /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    REGEXP_ESCAPE = / [bBdDsSwW] /x  # :nodoc:
     STRING_CONTENT_PATTERN = {
       "'" => /[^\\']+/,
       '"' => /[^\\"]+/,
@@ -100,7 +100,6 @@ module Scanners
             # TODO: scan over nested tags
             xml_scanner.tokenize match, :tokens => encoder
             value_expected = false
-            next
             
           elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
             value_expected = true

data/lib/coderay/scanners/lua.rb

@@ -76,7 +76,7 @@ module Scanners
             encoder.text_token(match, :comment)
           
           elsif match = scan(/\[=*\[/)     # [[ long (possibly multiline) string ]]
-            num_equals = match.count("=") # Number must match for comment end
+            num_equals = match.count("=") # Number must match for string end
             encoder.begin_group(:string)
             encoder.text_token(match, :delimiter)
             state = :long_string

data/lib/coderay/scanners/python.rb

@@ -63,7 +63,7 @@ module Scanners
     
     NAME = / [[:alpha:]_] \w* /x  # :nodoc:
     ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x  # :nodoc:
     
     OPERATOR = /
       \.\.\. |          # ellipsis

data/lib/coderay/scanners/ruby.rb

@@ -164,15 +164,19 @@ module Scanners
               end
               
             elsif match = scan(/ ' (?:(?>[^'\\]*) ')? | " (?:(?>[^"\\\#]*) ")? /mx)
-              encoder.begin_group :string
               if match.size == 1
+                kind = check(self.class::StringState.simple_key_pattern(match)) ? :key : :string
+                encoder.begin_group kind
                 encoder.text_token match, :delimiter
-                state = self.class::StringState.new :string, match == '"', match  # important for streaming
+                state = self.class::StringState.new kind, match == '"', match  # important for streaming
               else
+                kind = value_expected == true && scan(/:/) ? :key : :string
+                encoder.begin_group kind
                 encoder.text_token match[0,1], :delimiter
                 encoder.text_token match[1..-2], :content if match.size > 2
                 encoder.text_token match[-1,1], :delimiter
-                encoder.end_group :string
+                encoder.end_group kind
+                encoder.text_token ':', :operator if kind == :key
                 value_expected = false
               end
               
@@ -191,11 +195,14 @@ module Scanners
                 encoder.text_token match, :error
                 method_call_expected = false
               else
-                encoder.text_token match, self[1] ? :float : :integer  # TODO: send :hex/:octal/:binary
+                kind = self[1] ? :float : :integer  # TODO: send :hex/:octal/:binary
+                match << 'r' if match !~ /e/i && scan(/r/)
+                match << 'i' if scan(/i/)
+                encoder.text_token match, kind
               end
               value_expected = false
               
-            elsif match = scan(/ [-+!~^\/]=? | [:;] | [*|&]{1,2}=? | >>? /x)
+            elsif match = scan(/ [-+!~^\/]=? | [:;] | &\. | [*|&]{1,2}=? | >>? /x)
               value_expected = true
               encoder.text_token match, :operator
               
@@ -208,7 +215,7 @@ module Scanners
               encoder.end_group kind
               heredocs ||= []  # create heredocs if empty
               heredocs << self.class::StringState.new(kind, quote != "'", delim,
-                self[1] == '-' ? :indented : :linestart)
+                self[1] ? :indented : :linestart)
               value_expected = false
               
             elsif value_expected && match = scan(/#{patterns::FANCY_STRING_START}/o)

data/lib/coderay/scanners/ruby/patterns.rb

@@ -60,7 +60,7 @@ module Scanners
     
     QUOTE_TO_TYPE = {
       '`' => :shell,
-      '/'=> :regexp,
+      '/' => :regexp,
     }
     QUOTE_TO_TYPE.default = :string
     
@@ -114,7 +114,7 @@ module Scanners
     # NOTE: This is not completely correct, but
     # nobody needs heredoc delimiters ending with \n.
     HEREDOC_OPEN = /
-      << (-)?              # $1 = float
+      << ([-~])?           # $1 = float
       (?:
         ( [A-Za-z_0-9]+ )  # $2 = delim
       |

data/lib/coderay/scanners/ruby/string_state.rb

@@ -37,6 +37,14 @@ module Scanners
         end
       end
       
+      def self.simple_key_pattern delim
+        if delim == "'"
+          / (?> (?: [^\\']+ | \\. )* ) ' : /mx
+        else
+          / (?> (?: [^\\"\#]+ | \\. | \#\$[\\"] | \#\{[^\{\}]+\} | \#(?!\{) )* ) " : /mx
+        end
+      end
+      
       def initialize kind, interpreted, delim, heredoc = false
         if heredoc
           pattern = heredoc_pattern delim, interpreted, heredoc == :indented

data/lib/coderay/{scanner.rb → scanners/scanner.rb}

@@ -1,25 +1,7 @@
 # encoding: utf-8
-require 'strscan'
 
 module CodeRay
-  
-  autoload :WordList, coderay_path('helpers', 'word_list')
-  
-  # = Scanners
-  #
-  # This module holds the Scanner class and its subclasses.
-  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
-  # can be found in coderay/scanners/ruby.
-  #
-  # Scanner also provides methods and constants for the register
-  # mechanism and the [] method that returns the Scanner class
-  # belonging to the given lang.
-  #
-  # See PluginHost.
   module Scanners
-    extend PluginHost
-    plugin_path File.dirname(__FILE__), 'scanners'
-    
     
     # = Scanner
     #

data/lib/coderay/scanners/sql.rb

@@ -29,7 +29,7 @@ module Scanners
       char varchar varchar2 enum binary text tinytext mediumtext
       longtext blob tinyblob mediumblob longblob timestamp
       date time datetime year double decimal float int
-      integer tinyint mediumint bigint smallint unsigned bit
+      integer tinyint mediumint bigint smallint unsigned bit numeric
       bool boolean hex bin oct
     )
     
@@ -57,6 +57,12 @@ module Scanners
     
     STRING_PREFIXES = /[xnb]|_\w+/i
     
+    STRING_CONTENT_PATTERN = {
+      '"' => / (?: [^\\"] | "" )+ /x,
+      "'" => / (?: [^\\'] | '' )+ /x,
+      '`' => / (?: [^\\`] | `` )+ /x,
+    }
+    
     def scan_tokens encoder, options
       
       state = :initial
@@ -90,7 +96,7 @@ module Scanners
             state = :string
             encoder.text_token match, :delimiter
             
-          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9\$]* /x)
             encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
             name_expected = false
             
@@ -115,40 +121,26 @@ module Scanners
           end
           
         elsif state == :string
-          if match = scan(/[^\\"'`]+/)
-            string_content << match
-            next
+          if match = scan(STRING_CONTENT_PATTERN[string_type])
+            encoder.text_token match, :content
           elsif match = scan(/["'`]/)
             if string_type == match
               if peek(1) == string_type  # doubling means escape
-                string_content << string_type << getch
-                next
-              end
-              unless string_content.empty?
-                encoder.text_token string_content, :content
-                string_content = ''
+                encoder.text_token match + getch, :content
+              else
+                encoder.text_token match, :delimiter
+                encoder.end_group :string
+                state = :initial
+                string_type = nil
               end
-              encoder.text_token match, :delimiter
-              encoder.end_group :string
-              state = :initial
-              string_type = nil
             else
-              string_content << match
+              encoder.text_token match, :content
             end
           elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            unless string_content.empty?
-              encoder.text_token string_content, :content
-              string_content = ''
-            end
             encoder.text_token match, :char
           elsif match = scan(/ \\ . /mox)
-            string_content << match
-            next
+            encoder.text_token match, :content
           elsif match = scan(/ \\ | $ /x)
-            unless string_content.empty?
-              encoder.text_token string_content, :content
-              string_content = ''
-            end
             encoder.text_token match, :error unless match.empty?
             encoder.end_group :string
             state = :initial

data/lib/coderay/styles.rb

@@ -0,0 +1,15 @@
+module CodeRay
+
+  # This module holds the Style class and its subclasses.
+  # 
+  # See Plugin.
+  module Styles
+    
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'styles'
+    
+    autoload :Style, CodeRay.coderay_path('styles', 'style')
+    
+  end
+  
+end