coderay 1.0.9 → 1.1.0.rc1

data/lib/coderay/helpers/plugin.rb

@@ -131,7 +131,7 @@ module CodeRay
     
     # A Hash of plugion_id => Plugin pairs.
     def plugin_hash
-      @plugin_hash ||= make_plugin_hash
+      @plugin_hash ||= (@plugin_hash = make_plugin_hash).tap { load_plugin_map }
     end
     
     # Returns an array of all .rb files in the plugin path.
@@ -158,7 +158,6 @@ module CodeRay
     # This is done automatically when plugin_path is called.
     def load_plugin_map
       mapfile = path_to '_map'
-      @plugin_map_loaded = true
       if File.exist? mapfile
         require mapfile
         true
@@ -171,23 +170,16 @@ module CodeRay
     
     # Return a plugin hash that automatically loads plugins.
     def make_plugin_hash
-      @plugin_map_loaded ||= false
       Hash.new do |h, plugin_id|
         id = validate_id(plugin_id)
         path = path_to id
         begin
           require path
         rescue LoadError => boom
-          if @plugin_map_loaded
-            if h.has_key?(:default)
-              warn '%p could not load plugin %p; falling back to %p' % [self, id, h[:default]]
-              h[:default]
-            else
-              raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
-            end
+          if h.has_key?(:default)
+            h[:default]
           else
-            load_plugin_map
-            h[plugin_id]
+            raise PluginNotFound, '%p could not load plugin %p: %s' % [self, id, boom]
           end
         else
           # Plugin should have registered by now
@@ -271,7 +263,6 @@ module CodeRay
     end
     
     def aliases
-      plugin_host.load_plugin_map
       plugin_host.plugin_hash.inject [] do |aliases, (key, _)|
         aliases << key if plugin_host[key] == self
         aliases

data/lib/coderay/scanner.rb

@@ -182,16 +182,9 @@ module CodeRay
       # Scan the code and returns all tokens in a Tokens object.
       def tokenize source = nil, options = {}
         options = @options.merge(options)
-        @tokens = options[:tokens] || @tokens || Tokens.new
-        @tokens.scanner = self if @tokens.respond_to? :scanner=
-        case source
-        when Array
-          self.string = self.class.normalize(source.join)
-        when nil
-          reset
-        else
-          self.string = self.class.normalize(source)
-        end
+        
+        set_tokens_from_options options
+        set_string_from_source source
         
         begin
           scan_tokens @tokens, options
@@ -261,6 +254,22 @@ module CodeRay
       def setup  # :doc:
       end
       
+      def set_string_from_source source
+        case source
+        when Array
+          self.string = self.class.normalize(source.join)
+        when nil
+          reset
+        else
+          self.string = self.class.normalize(source)
+        end
+      end
+      
+      def set_tokens_from_options options
+        @tokens = options[:tokens] || @tokens || Tokens.new
+        @tokens.scanner = self if @tokens.respond_to? :scanner=
+      end
+      
       # This is the central method, and commonly the only one a
       # subclass implements.
       #
@@ -277,19 +286,15 @@ module CodeRay
         @binary_string = nil if defined? @binary_string
       end
       
-      # Scanner error with additional status information
-      def raise_inspect msg, tokens, state = self.state || 'No state given!', ambit = 30, backtrace = caller
-        raise ScanError, <<-EOE % [
+      SCAN_ERROR_MESSAGE = <<-MESSAGE
 
 
-***ERROR in %s: %s (after %d tokens)
+***ERROR in %s: %s (after %s tokens)
 
 tokens:
 %s
 
-current line: %d  column: %d  pos: %d
-matched: %p  state: %p
-bol? = %p,  eos? = %p
+%s
 
 surrounding code:
 %p  ~~  %p
@@ -297,16 +302,43 @@ surrounding code:
 
 ***ERROR***
 
-        EOE
-          File.basename(caller[0]),
-          msg,
-          tokens.respond_to?(:size) ? tokens.size : 0,
-          tokens.respond_to?(:last) ? tokens.last(10).map { |t| t.inspect }.join("\n") : '',
-          line, column, pos,
-          matched, state, bol?, eos?,
+      MESSAGE
+      
+      def raise_inspect_arguments message, tokens, state, ambit
+        return File.basename(caller[0]),
+          message,
+          tokens_size(tokens),
+          tokens_last(tokens, 10).map(&:inspect).join("\n"),
+          scanner_state_info(state),
           binary_string[pos - ambit, ambit],
-          binary_string[pos, ambit],
-        ], backtrace
+          binary_string[pos, ambit]
+      end
+      
+      SCANNER_STATE_INFO = <<-INFO
+current line: %d  column: %d  pos: %d
+matched: %p  state: %p
+bol?: %p,  eos?: %p
+      INFO
+      
+      def scanner_state_info state
+        SCANNER_STATE_INFO % [
+          line, column, pos,
+          matched, state || 'No state given!',
+          bol?, eos?,
+        ]
+      end
+      
+      # Scanner error with additional status information
+      def raise_inspect message, tokens, state = self.state, ambit = 30, backtrace = caller
+        raise ScanError, SCAN_ERROR_MESSAGE % raise_inspect_arguments(message, tokens, state, ambit), backtrace
+      end
+      
+      def tokens_size tokens
+        tokens.size if tokens.respond_to?(:size)
+      end
+      
+      def tokens_last tokens, n
+        tokens.respond_to?(:last) ? tokens.last(n) : []
       end
       
       # Shorthand for scan_until(/\z/).

data/lib/coderay/scanners/c.rb

@@ -148,7 +148,7 @@ module Scanners
             encoder.text_token match, :char
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group :string
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else

data/lib/coderay/scanners/cpp.rb

@@ -160,7 +160,7 @@ module Scanners
             encoder.text_token match, :char
           elsif match = scan(/ \\ | $ /x)
             encoder.end_group :string
-            encoder.text_token match, :error
+            encoder.text_token match, :error unless match.empty?
             state = :initial
             label_expected = false
           else

data/lib/coderay/scanners/css.rb

@@ -7,27 +7,25 @@ module Scanners
     
     KINDS_NOT_LOC = [
       :comment,
-      :class, :pseudo_class, :type,
-      :constant, :directive,
+      :class, :pseudo_class, :tag,
+      :id, :directive,
       :key, :value, :operator, :color, :float, :string,
-      :error, :important,
+      :error, :important, :type,
     ]  # :nodoc:
     
     module RE  # :nodoc:
       Hex = /[0-9a-fA-F]/
-      Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
-      Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
-      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
-      NMStart = /[_a-zA-Z]|#{Escape}/
-      NL = /\r\n|\r|\n|\f/
-      String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # TODO: buggy regexp
-      String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # TODO: buggy regexp
+      Unicode = /\\#{Hex}{1,6}\b/ # differs from standard because it allows uppercase hex too
+      Escape = /#{Unicode}|\\[^\n0-9a-fA-F]/
+      NMChar = /[-_a-zA-Z0-9]/
+      NMStart = /[_a-zA-Z]/
+      String1 = /"(?:[^\n\\"]+|\\\n|#{Escape})*"?/  # TODO: buggy regexp
+      String2 = /'(?:[^\n\\']+|\\\n|#{Escape})*'?/  # TODO: buggy regexp
       String = /#{String1}|#{String2}/
       
       HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
-      Color = /#{HexColor}/
       
-      Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
+      Num = /-?(?:[0-9]*\.[0-9]+|[0-9]+)n?/
       Name = /#{NMChar}+/
       Ident = /-?#{NMStart}#{NMChar}*/
       AtKeyword = /@#{Ident}/
@@ -35,16 +33,15 @@ module Scanners
       
       reldimensions = %w[em ex px]
       absdimensions = %w[in cm mm pt pc]
-      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s]))
+      Unit = Regexp.union(*(reldimensions + absdimensions + %w[s dpi dppx deg]))
       
       Dimension = /#{Num}#{Unit}/
       
-      Comment = %r! /\* (?: .*? \*/ | .* ) !mx
-      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n\r\f]|\\\))*\)?/
+      Function = /(?:url|alpha|attr|counters?)\((?:[^)\n]|\\\))*\)?/
       
-      Id = /##{Name}/
+      Id = /(?!#{HexColor}\b(?!-))##{Name}/
       Class = /\.#{Name}/
-      PseudoClass = /:#{Name}/
+      PseudoClass = /::?#{Ident}/
       AttributeSelector = /\[[^\]]*\]?/
     end
     
@@ -52,11 +49,11 @@ module Scanners
     
     def setup
       @state = :initial
-      @value_expected = nil
+      @value_expected = false
     end
     
     def scan_tokens encoder, options
-      states = Array(options[:state] || @state)
+      states = Array(options[:state] || @state).dup
       value_expected = @value_expected
       
       until eos?
@@ -67,13 +64,13 @@ module Scanners
         elsif case states.last
           when :initial, :media
             if match = scan(/(?>#{RE::Ident})(?!\()|\*/ox)
-              encoder.text_token match, :type
+              encoder.text_token match, :tag
               next
             elsif match = scan(RE::Class)
               encoder.text_token match, :class
               next
             elsif match = scan(RE::Id)
-              encoder.text_token match, :constant
+              encoder.text_token match, :id
               next
             elsif match = scan(RE::PseudoClass)
               encoder.text_token match, :pseudo_class
@@ -148,17 +145,17 @@ module Scanners
           start = match[/^\w+\(/]
           encoder.text_token start, :delimiter
           if match[-1] == ?)
-            encoder.text_token match[start.size..-2], :content
+            encoder.text_token match[start.size..-2], :content if match.size > start.size + 1
             encoder.text_token ')', :delimiter
           else
-            encoder.text_token match[start.size..-1], :content
+            encoder.text_token match[start.size..-1], :content if match.size > start.size
           end
           encoder.end_group :function
           
         elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
           encoder.text_token match, :float
           
-        elsif match = scan(/#{RE::Color}/o)
+        elsif match = scan(/#{RE::HexColor}/o)
           encoder.text_token match, :color
           
         elsif match = scan(/! *important/)
@@ -170,7 +167,7 @@ module Scanners
         elsif match = scan(RE::AtKeyword)
           encoder.text_token match, :directive
           
-        elsif match = scan(/ [+>:;,.=()\/] /x)
+        elsif match = scan(/ [+>~:;,.=()\/] /x)
           if match == ':'
             value_expected = true
           elsif match == ';'

data/lib/coderay/scanners/diff.rb

@@ -20,7 +20,7 @@ module Scanners
       
       line_kind = nil
       state = :initial
-      deleted_lines = 0
+      deleted_lines_count = 0
       scanners = Hash.new do |h, lang|
         h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
       end
@@ -30,7 +30,7 @@ module Scanners
       until eos?
         
         if match = scan(/\n/)
-          deleted_lines = 0 unless line_kind == :delete
+          deleted_lines_count = 0 unless line_kind == :delete
           if line_kind
             encoder.end_line line_kind
             line_kind = nil
@@ -45,7 +45,7 @@ module Scanners
           if match = scan(/--- |\+\+\+ |=+|_+/)
             encoder.begin_line line_kind = :head
             encoder.text_token match, :head
-            if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
+            if match = scan(/[^\x00\n]+?(?=$|[\t\n]|  \(revision)/)
               encoder.text_token match, :filename
               if options[:highlight_code] && match != '/dev/null'
                 file_type = CodeRay::FileType.fetch(match, :text)
@@ -69,7 +69,7 @@ module Scanners
             state = :added
           elsif match = scan(/\\ .*/)
             encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]*)@@/)
+          elsif match = scan(/@@(?>[^@\n]+)@@/)
             content_scanner.state = :initial unless match?(/\n\+/)
             content_scanner_entry_state = nil
             if check(/\n|$/)
@@ -99,37 +99,59 @@ module Scanners
             end
             next
           elsif match = scan(/-/)
-            deleted_lines += 1
-            encoder.begin_line line_kind = :delete
-            encoder.text_token match, :delete
-            if options[:inline_diff] && deleted_lines == 1 && check(/(?>.*)\n\+(?>.*)$(?!\n\+)/)
-              content_scanner_entry_state = content_scanner.state
-              skip(/(.*)\n\+(.*)$/)
-              head, deletion, insertion, tail = diff self[1], self[2]
-              pre, deleted, post = content_scanner.tokenize [head, deletion, tail], :tokens => Tokens.new
-              encoder.tokens pre
-              unless deleted.empty?
-                encoder.begin_group :eyecatcher
-                encoder.tokens deleted
-                encoder.end_group :eyecatcher
+            deleted_lines_count += 1
+            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
+              deleted_lines  = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) }
+              inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/)         ; scan(/.*/) }
+              
+              deleted_lines_tokenized  = []
+              inserted_lines_tokenized = []
+              for deleted_line, inserted_line in deleted_lines.zip(inserted_lines)
+                pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line
+                content_scanner_entry_state = content_scanner.state
+                deleted_lines_tokenized  << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new)
+                content_scanner.state = content_scanner_entry_state || :initial
+                inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new)
               end
-              encoder.tokens post
-              encoder.end_line line_kind
-              encoder.text_token "\n", :space
-              encoder.begin_line line_kind = :insert
-              encoder.text_token '+', :insert
-              content_scanner.state = content_scanner_entry_state || :initial
-              pre, inserted, post = content_scanner.tokenize [head, insertion, tail], :tokens => Tokens.new
-              encoder.tokens pre
-              unless inserted.empty?
-                encoder.begin_group :eyecatcher
-                encoder.tokens inserted
-                encoder.end_group :eyecatcher
+              
+              for pre, deleted_part, post in deleted_lines_tokenized
+                encoder.begin_line :delete
+                encoder.text_token '-', :delete
+                encoder.tokens pre
+                unless deleted_part.empty?
+                  encoder.begin_group :eyecatcher
+                  encoder.tokens deleted_part
+                  encoder.end_group :eyecatcher
+                end
+                encoder.tokens post
+                encoder.end_line :delete
+                encoder.text_token "\n", :space
+              end
+              
+              for pre, inserted_part, post in inserted_lines_tokenized
+                encoder.begin_line :insert
+                encoder.text_token '+', :insert
+                encoder.tokens pre
+                unless inserted_part.empty?
+                  encoder.begin_group :eyecatcher
+                  encoder.tokens inserted_part
+                  encoder.end_group :eyecatcher
+                end
+                encoder.tokens post
+                changed_lines_count -= 1
+                if changed_lines_count > 0
+                  encoder.end_line :insert
+                  encoder.text_token "\n", :space
+                end
               end
-              encoder.tokens post
+              
+              line_kind = :insert
+              
             elsif match = scan(/.*/)
+              encoder.begin_line line_kind = :delete
+              encoder.text_token '-', :delete
               if options[:highlight_code]
-                if deleted_lines == 1
+                if deleted_lines_count == 1
                   content_scanner_entry_state = content_scanner.state
                 end
                 content_scanner.tokenize match, :tokens => encoder unless match.empty?

data/lib/coderay/scanners/groovy.rb

@@ -36,9 +36,12 @@ module Scanners
     
   protected
     
+    def setup
+      @state = :initial
+    end
+    
     def scan_tokens encoder, options
-      
-      state = :initial
+      state = options[:state] || @state
       inline_block_stack = []
       inline_block_paren_depth = nil
       string_delimiter = nil
@@ -223,7 +226,7 @@ module Scanners
             encoder.text_token match, :content  # TODO: Shouldn't this be :error?
             
           elsif match = scan(/ \\ | \n /x)
-            encoder.end_group state
+            encoder.end_group state == :regexp ? :regexp : :string
             encoder.text_token match, :error
             after_def = value_expected = false
             state = :initial
@@ -243,7 +246,17 @@ module Scanners
       end
       
       if [:multiline_string, :string, :regexp].include? state
-        encoder.end_group state
+        encoder.end_group state == :regexp ? :regexp : :string
+      end
+      
+      if options[:keep_state]
+        @state = state
+      end
+      
+      until inline_block_stack.empty?
+        state, = *inline_block_stack.pop
+        encoder.end_group :inline
+        encoder.end_group state == :regexp ? :regexp : :string
       end
       
       encoder