coderay 1.0.0.800pre → 1.0.0.815pre

data/lib/coderay/helpers/file_type.rb

@@ -90,7 +90,7 @@ module CodeRay
       'h'        => :c,
       'htm'      => :page,
       'html'     => :page,
-      'html.erb' => :rhtml,
+      'html.erb' => :erb,
       'java'     => :java,
       'js'       => :java_script,
       'json'     => :json,
@@ -109,14 +109,14 @@ module CodeRay
       'raydebug' => :raydebug,
       'rb'       => :ruby,
       'rbw'      => :ruby,
-      'rhtml'    => :rhtml,
+      'rhtml'    => :erb,
       'rjs'      => :ruby,
       'rpdf'     => :ruby,
       'ru'       => :ruby,
       'rxml'     => :ruby,
-      'sch'      => :scheme,
+      # 'sch'      => :scheme,
       'sql'      => :sql,
-      'ss'       => :scheme,
+      # 'ss'       => :scheme,
       'xhtml'    => :page,
       'xml'      => :xml,
       'yaml'     => :yaml,

data/lib/coderay/helpers/plugin.rb

@@ -114,9 +114,10 @@ module CodeRay
     def default id = nil
       if id
         id = validate_id id
-        plugin_hash[nil] = id
+        raise "The default plugin can't be named \"default\"." if id == :default
+        plugin_hash[:default] = id
       else
-        load nil
+        load :default
       end
     end
     
@@ -179,10 +180,11 @@ module CodeRay
           require path
         rescue LoadError => boom
           if @plugin_map_loaded
-            if h.has_key?(nil)  # default plugin
-              h[nil]
+            if h.has_key?(:default)
+              warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]]
+              h[:default]
             else
-              raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
+              raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
             end
           else
             load_plugin_map

data/lib/coderay/scanner.rb

@@ -74,7 +74,7 @@ module CodeRay
           if code.respond_to? :encoding
             code = encode_with_encoding code, self.encoding
           else
-            code = to_unix code if code.index ?\r
+            code = to_unix code
           end
           # code = code.dup if code.eql? original
           code
@@ -100,7 +100,7 @@ module CodeRay
         def encode_with_encoding code, target_encoding
           if code.encoding == target_encoding
             if code.valid_encoding?
-              return to_unix(code)
+              return to_unix code
             else
               source_encoding = guess_encoding code
             end
@@ -112,7 +112,7 @@ module CodeRay
         end
         
         def to_unix code
-          code.gsub(/\r\n?/, "\n")
+          code.index(?\r) ? code.gsub(/\r\n?/, "\n") : code
         end
         
         def guess_encoding s
@@ -221,27 +221,39 @@ module CodeRay
       end
       include Enumerable
       
-      # The current line position of the scanner. See also #column.
+      # The current line position of the scanner, starting with 1.
+      # See also: #column.
       #
       # Beware, this is implemented inefficiently. It should be used
       # for debugging only.
-      def line
-        string[0..pos].count("\n") + 1
+      def line pos = self.pos
+        return 1 if pos <= 0
+        binary_string[0...pos].count("\n") + 1
       end
       
-      # The current column position of the scanner. See also #line.
+      # The current column position of the scanner, starting with 1.
+      # See also: #line.
       #
       # Beware, this is implemented inefficiently. It should be used
       # for debugging only.
       def column pos = self.pos
-        return 0 if pos <= 0
-        string = self.string
-        if string.respond_to?(:bytesize) && string.bytesize != string.size
-          #:nocov:
-          string = string.dup.force_encoding('binary')
-          #:nocov:
-        end
-        pos - (string.rindex(?\n, pos) || 0)
+        return 1 if pos <= 0
+        pos - (binary_string.rindex(?\n, pos - 1) || -1)
+      end
+      
+      # The string in binary encoding.
+      # 
+      # To be used with #pos, which is the index of the byte the scanner
+      # will scan next.
+      def binary_string
+        @binary_string ||=
+          if string.respond_to?(:bytesize) && string.bytesize != string.size
+            #:nocov:
+            string.dup.force_encoding('binary')
+            #:nocov:
+          else
+            string
+          end
       end
       
     protected
@@ -267,7 +279,7 @@ module CodeRay
       def reset_instance
         @tokens.clear if @tokens.respond_to?(:clear) && !@options[:keep_tokens]
         @cached_tokens = nil
-        @bin_string = nil if defined? @bin_string
+        @binary_string = nil if defined? @binary_string
       end
       
       # Scanner error with additional status information
@@ -297,8 +309,8 @@ surrounding code:
           tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
           line, column, pos,
           matched, state, bol?, eos?,
-          string[pos - ambit, ambit],
-          string[pos, ambit],
+          binary_string[pos - ambit, ambit],
+          binary_string[pos, ambit],
         ], backtrace
       end
       

data/lib/coderay/scanners/_map.rb

@@ -2,20 +2,21 @@ module CodeRay
 module Scanners
   
   map \
-    :cplusplus => :cpp,
-    :'c++' => :cpp,
-    :ecmascript => :java_script,
+    :'c++'       => :cpp,
+    :cplusplus   => :cpp,
+    :ecmascript  => :java_script,
     :ecma_script => :java_script,
-    :irb => :ruby,
-    :javascript => :java_script,
-    :js => :java_script,
-    :nitro => :nitro_xhtml,
-    :pascal => :delphi,
-    :patch => :diff,
-    :plain => :text,
-    :plaintext => :text,
-    :xhtml => :html,
-    :yml => :yaml
+    :rhtml       => :erb,
+    :eruby       => :erb,
+    :irb         => :ruby,
+    :javascript  => :java_script,
+    :js          => :java_script,
+    :pascal      => :delphi,
+    :patch       => :diff,
+    :plain       => :text,
+    :plaintext   => :text,
+    :xhtml       => :html,
+    :yml         => :yaml
   
   default :text
   

data/lib/coderay/scanners/clojure.rb

@@ -156,7 +156,7 @@ module CodeRay
             elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|[@\^]/)
               encoder.text_token match, :operator
             elsif match = scan(/;.*/)
-              encoder.text_token match, :comment  # FIXME: recognize (comment ...) too
+              encoder.text_token match, :comment  # TODO: recognize (comment ...) too
             elsif match = scan(/\#?\\(?:newline|space|.?)/)
               encoder.text_token match, :char
             elsif match = scan(/\#[ft]/)

data/lib/coderay/scanners/css.rb

@@ -2,9 +2,9 @@ module CodeRay
 module Scanners
 
   class CSS < Scanner
-
+    
     register_for :css
-
+    
     KINDS_NOT_LOC = [
       :comment,
       :class, :pseudo_class, :type,
@@ -20,28 +20,28 @@ module Scanners
       NMChar = /[-_a-zA-Z0-9]|#{Escape}/
       NMStart = /[_a-zA-Z]|#{Escape}/
       NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp
+      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
       String = /#{String1}|#{String2}/
-
+      
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
       Color = /#{HexColor}/
-
+      
       Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
       Percentage = /#{Num}%/
-
+      
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
       Unit = Regexp.union(*(reldimensions + absdimensions))
-
+      
       Dimension = /#{Num}#{Unit}/
-
+      
       Comment = %r! /\* (?: .*? \*/ | .* ) !mx
       Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
-
+      
       Id = /##{Name}/
       Class = /\.#{Name}/
       PseudoClass = /:#{Name}/
@@ -64,20 +64,26 @@ module Scanners
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
               encoder.text_token match, :type
+              next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
+              next
             elsif match = scan(RE::Id)
               encoder.text_token match, :constant
+              next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
+              next
             elsif match = scan(RE::AttributeSelector)
               # TODO: Improve highlighting inside of attribute selectors.
               encoder.text_token match[0,1], :operator
               encoder.text_token match[1..-2], :attribute_name if match.size > 2
               encoder.text_token match[-1,1], :operator if match[-1] == ?]
+              next
             elsif match = scan(/@media/)
               encoder.text_token match, :directive
               states.push :media_before_name
+              next
             end
           
           when :block
@@ -87,18 +93,21 @@ module Scanners
               else
                 encoder.text_token match, :key
               end
+              next
             end
-
+            
           when :media_before_name
             if match = scan(RE::Ident)
               encoder.text_token match, :type
               states[-1] = :media_after_name
+              next
             end
           
           when :media_after_name
             if match = scan(/\{/)
               encoder.text_token match, :operator
               states[-1] = :media
+              next
             end
           
           else
@@ -110,12 +119,12 @@ module Scanners
           
         elsif match = scan(/\/\*(?:.*?\*\/|\z)/m)
           encoder.text_token match, :comment
-
+          
         elsif match = scan(/\{/)
           value_expected = false
           encoder.text_token match, :operator
           states.push :block
-
+          
         elsif match = scan(/\}/)
           value_expected = false
           if states.last == :block || states.last == :media
@@ -124,14 +133,14 @@ module Scanners
           else
             encoder.text_token match, :error
           end
-
+          
         elsif match = scan(/#{RE::String}/o)
           encoder.begin_group :string
           encoder.text_token match[0, 1], :delimiter
           encoder.text_token match[1..-2], :content if match.size > 2
           encoder.text_token match[-1, 1], :delimiter if match.size >= 2
           encoder.end_group :string
-
+          
         elsif match = scan(/#{RE::Function}/o)
           encoder.begin_group :string
           start = match[/^\w+\(/]
@@ -143,22 +152,22 @@ module Scanners
             encoder.text_token match[start.size..-1], :content
           end
           encoder.end_group :string
-
+          
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
-
+          
         elsif match = scan(/#{RE::Color}/o)
           encoder.text_token match, :color
-
+          
         elsif match = scan(/! *important/)
           encoder.text_token match, :important
-
+          
         elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
           encoder.text_token match, :color
-
+          
         elsif match = scan(RE::AtKeyword)
           encoder.text_token match, :directive
-
+          
         elsif match = scan(/ [+>:;,.=()\/] /x)
           if match == ':'
             value_expected = true
@@ -166,18 +175,18 @@ module Scanners
             value_expected = false
           end
           encoder.text_token match, :operator
-
+          
         else
           encoder.text_token getch, :error
-
+          
         end
-
+        
       end
-
+      
       encoder
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/scanners/{rhtml.rb → erb.rb}

@@ -5,9 +5,9 @@ module Scanners
   load :ruby
   
   # Scanner for HTML ERB templates.
-  class RHTML < Scanner
+  class ERB < Scanner
     
-    register_for :rhtml
+    register_for :erb
     title 'HTML ERB Template'
     
     KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
@@ -56,7 +56,7 @@ module Scanners
           if start_tag[/\A<%#/]
             encoder.text_token code, :comment
           else
-            @ruby_scanner.tokenize code
+            @ruby_scanner.tokenize code, :tokens => encoder
           end unless code.empty?
           encoder.text_token end_tag, :inline_delimiter unless end_tag.empty?
           encoder.end_group :inline

data/lib/coderay/scanners/groovy.rb

@@ -220,7 +220,7 @@ module Scanners
             encoder.text_token match, :content
             
           elsif match = scan(/ \\. /mx)
-            encoder.text_token match, :content  # FIXME: Shouldn't this be :error?
+            encoder.text_token match, :content  # TODO: Shouldn't this be :error?
             
           elsif match = scan(/ \\ | \n /x)
             encoder.end_group state

data/lib/coderay/scanners/java_script.rb

@@ -89,7 +89,7 @@ module Scanners
             end
             
           elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
-            # FIXME: scan over nested tags
+            # TODO: scan over nested tags
             xml_scanner.tokenize match, :tokens => encoder
             value_expected = false
             next

data/lib/coderay/scanners/php.rb

@@ -234,8 +234,8 @@ module Scanners
     def scan_tokens encoder, options
       
       if check(RE::PHP_START) ||  # starts with <?
-       (match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
-       exist?(RE::HTML_INDICATOR) ||
+       (match?(/\s*<\S/) && check(/.{1,1000}#{RE::PHP_START}/om)) || # starts with tag and contains <?
+       check(/.{0,1000}#{RE::HTML_INDICATOR}/om) ||
        check(/.{1,100}#{RE::PHP_START}/om)  # PHP start after max 100 chars
         # is HTML with embedded PHP, so start with HTML
         states = [:initial]

data/lib/coderay/scanners/ruby.rb

@@ -26,7 +26,7 @@ module Scanners
       state, heredocs = @state
       heredocs = heredocs.dup if heredocs.is_a?(Array)
       
-      if state && state.instance_of?(self.class::StringState)
+      if state && state.instance_of?(StringState)
         encoder.begin_group state.type
       end
       
@@ -426,13 +426,18 @@ module Scanners
       end
       
       # cleaning up
-      if options[:keep_state]
-        heredocs = nil if heredocs && heredocs.empty?
-        @state = state, heredocs
+      if state.is_a? StringState
+        encoder.end_group state.type
       end
       
-      if state.is_a? self.class::StringState
-        encoder.end_group state.type
+      if options[:keep_state]
+        if state.is_a?(StringState) && state.heredoc
+          (heredocs ||= []).unshift state
+          state = :initial
+        elsif heredocs && heredocs.empty?
+          heredocs = nil
+        end
+        @state = state, heredocs
       end
       
       if inline_block_stack