coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/debug.rb

@@ -1,62 +1,65 @@
 module CodeRay
 module Scanners
-
+  
   # = Debug Scanner
+  # 
+  # Interprets the output of the Encoders::Debug encoder.
   class Debug < Scanner
-
-    include Streamable
+    
     register_for :debug
-    file_extension 'raydebug'
-    title 'CodeRay Token Dump'
-
+    title 'CodeRay Token Dump Import'
+    
   protected
-    def scan_tokens tokens, options
-
+    
+    def scan_tokens encoder, options
+      
       opened_tokens = []
-
+      
       until eos?
-
-        kind = nil
-        match = nil
-
-          if scan(/\s+/)
-            tokens << [matched, :space]
-            next
-            
-          elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
-            kind = self[1].to_sym
-            match = self[2].gsub(/\\(.)/, '\1')
-            
-          elsif scan(/ (\w+) < /x)
-            kind = self[1].to_sym
-            opened_tokens << kind
-            match = :open
-            
-          elsif !opened_tokens.empty? && scan(/ > /x)
-            kind = opened_tokens.pop || :error
-            match = :close
-            
-          else
+        
+        if match = scan(/\s+/)
+          encoder.text_token match, :space
+          
+        elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
+          kind = self[1].to_sym
+          match = self[2].gsub(/\\(.)/m, '\1')
+          unless TokenKinds.has_key? kind
             kind = :error
-            getch
-
+            match = matched
           end
-                  
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+          encoder.text_token match, kind
+          
+        elsif match = scan(/ (\w+) ([<\[]) /x)
+          kind = self[1].to_sym
+          opened_tokens << kind
+          case self[2]
+          when '<'
+            encoder.begin_group kind
+          when '['
+            encoder.begin_line kind
+          else
+            raise 'CodeRay bug: This case should not be reached.'
+          end
+          
+        elsif !opened_tokens.empty? && match = scan(/ > /x)
+          encoder.end_group opened_tokens.pop
+          
+        elsif !opened_tokens.empty? && match = scan(/ \] /x)
+          encoder.end_line opened_tokens.pop
+          
+        else
+          encoder.text_token getch, :space
+          
         end
-        raise_inspect 'Empty token', tokens unless match
-
-        tokens << [match, kind]
         
       end
       
-      tokens
+      encoder.end_group opened_tokens.pop until opened_tokens.empty?
+      
+      encoder
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/scanners/delphi.rb

@@ -1,12 +1,15 @@
 module CodeRay
 module Scanners
   
+  # Scanner for the Delphi language (Object Pascal).
+  # 
+  # Alias: +pascal+
   class Delphi < Scanner
-
+    
     register_for :delphi
     file_extension 'pas'
     
-    RESERVED_WORDS = [
+    KEYWORDS = [
       'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
       'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
       'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
@@ -16,9 +19,9 @@ module Scanners
       'procedure', 'program', 'property', 'raise', 'record', 'repeat',
       'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
       'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
-      'xor', 'on'
-    ]
-
+      'xor', 'on',
+    ]  # :nodoc:
+    
     DIRECTIVES = [
       'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
       'contains', 'deprecated', 'dispid', 'dynamic', 'export',
@@ -27,121 +30,112 @@ module Scanners
       'package', 'pascal', 'platform', 'private', 'protected', 'public',
       'published', 'read', 'readonly', 'register', 'reintroduce',
       'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
-      'virtual', 'write', 'writeonly'
-    ]
-
-    IDENT_KIND = CaseIgnoringWordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(DIRECTIVES, :directive)
+      'virtual', 'write', 'writeonly',
+    ]  # :nodoc:
     
-    NAME_FOLLOWS = CaseIgnoringWordList.new(false).
-      add(%w(procedure function .))
-
-  private
-    def scan_tokens tokens, options
-
+    IDENT_KIND = WordList::CaseIgnoring.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(DIRECTIVES, :directive)  # :nodoc:
+    
+    NAME_FOLLOWS = WordList::CaseIgnoring.new(false).
+      add(%w(procedure function .))  # :nodoc:
+    
+  protected
+    
+    def scan_tokens encoder, options
+      
       state = :initial
       last_token = ''
-
+      
       until eos?
-
-        kind = nil
-        match = nil
-
+        
         if state == :initial
           
-          if scan(/ \s+ /x)
-            tokens << [matched, :space]
+          if match = scan(/ \s+ /x)
+            encoder.text_token match, :space
             next
             
-          elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
-            tokens << [matched, :preprocessor]
+          elsif match = scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
+            encoder.text_token match, :preprocessor
             next
             
-          elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
-            tokens << [matched, :comment]
+          elsif match = scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
+            encoder.text_token match, :comment
             next
             
           elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
-            kind = :operator
+            encoder.text_token match, :operator
           
           elsif match = scan(/\./)
-            kind = :operator
-            if last_token == 'end'
-              tokens << [match, kind]
-              next
-            end
+            encoder.text_token match, :operator
+            next if last_token == 'end'
             
           elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
-            kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
+            encoder.text_token match, NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
             
-          elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
-            tokens << [:open, :char]
-            tokens << ["'", :delimiter]
-            tokens << [self[1], :content]
-            tokens << ["'", :delimiter]
-            tokens << [:close, :char]
+          elsif match = skip(/ ' ( [^\n']|'' ) (?:'|$) /x)
+            encoder.begin_group :char
+            encoder.text_token "'", :delimiter
+            encoder.text_token self[1], :content
+            encoder.text_token "'", :delimiter
+            encoder.end_group :char
             next
             
           elsif match = scan(/ ' /x)
-            tokens << [:open, :string]
+            encoder.begin_group :string
+            encoder.text_token match, :delimiter
             state = :string
-            kind = :delimiter
             
-          elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
-            kind = :char
+          elsif match = scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
+            encoder.text_token match, :char
             
-          elsif scan(/ \$ [0-9A-Fa-f]+ /x)
-            kind = :hex
+          elsif match = scan(/ \$ [0-9A-Fa-f]+ /x)
+            encoder.text_token match, :hex
             
-          elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
-            kind = :integer
+          elsif match = scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
+            encoder.text_token match, :integer
+            
+          elsif match = scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
+            encoder.text_token match, :float
             
-          elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
-            kind = :float
-
           else
-            kind = :error
-            getch
-
+            encoder.text_token getch, :error
+            next
+            
           end
           
         elsif state == :string
-          if scan(/[^\n']+/)
-            kind = :content
-          elsif scan(/''/)
-            kind = :char
-          elsif scan(/'/)
-            tokens << ["'", :delimiter]
-            tokens << [:close, :string]
+          if match = scan(/[^\n']+/)
+            encoder.text_token match, :content
+          elsif match = scan(/''/)
+            encoder.text_token match, :char
+          elsif match = scan(/'/)
+            encoder.text_token match, :delimiter
+            encoder.end_group :string
             state = :initial
             next
-          elsif scan(/\n/)
-            tokens << [:close, :string]
-            kind = :error
+          elsif match = scan(/\n/)
+            encoder.end_group :string
+            encoder.text_token match, :space
             state = :initial
           else
-            raise "else case \' reached; %p not handled." % peek(1), tokens
+            raise "else case \' reached; %p not handled." % peek(1), encoder
           end
           
         else
-          raise 'else-case reached', tokens
+          raise 'else-case reached', encoder
           
         end
         
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, state
-        end
-        raise_inspect 'Empty token', tokens unless match
-
         last_token = match
-        tokens << [match, kind]
         
       end
       
-      tokens
+      if state == :string
+        encoder.end_group state
+      end
+      
+      encoder
     end
 
   end

data/lib/coderay/scanners/diff.rb

@@ -1,25 +1,43 @@
 module CodeRay
 module Scanners
   
+  # Scanner for output of the diff command.
+  # 
+  # Alias: +patch+
   class Diff < Scanner
     
     register_for :diff
     title 'diff output'
     
-    def scan_tokens tokens, options
+    DEFAULT_OPTIONS = {
+      :highlight_code => true,
+      :inline_diff    => true,
+    }
+    
+  protected
+    
+    require 'coderay/helpers/file_type'
+    
+    def scan_tokens encoder, options
       
       line_kind = nil
       state = :initial
+      deleted_lines = 0
+      scanners = Hash.new do |h, lang|
+        h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
+      end
+      content_scanner = scanners[:plain]
+      content_scanner_entry_state = nil
       
       until eos?
-        kind = match = nil
         
         if match = scan(/\n/)
+          deleted_lines = 0 unless line_kind == :delete
           if line_kind
-            tokens << [:end_line, line_kind]
+            encoder.end_line line_kind
             line_kind = nil
           end
-          tokens << [match, :space]
+          encoder.text_token match, :space
           next
         end
         
@@ -27,81 +45,154 @@ module Scanners
         
         when :initial
           if match = scan(/--- |\+\+\+ |=+|_+/)
-            tokens << [:begin_line, line_kind = :head]
-            tokens << [match, :head]
+            encoder.begin_line line_kind = :head
+            encoder.text_token match, :head
+            if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
+              encoder.text_token match, :filename
+              if options[:highlight_code]
+                file_type = FileType.fetch(match, :text)
+                file_type = :text if file_type == :diff
+                content_scanner = scanners[file_type]
+                content_scanner_entry_state = nil
+              end
+            end
             next unless match = scan(/.+/)
-            kind = :plain
+            encoder.text_token match, :plain
           elsif match = scan(/Index: |Property changes on: /)
-            tokens << [:begin_line, line_kind = :head]
-            tokens << [match, :head]
+            encoder.begin_line line_kind = :head
+            encoder.text_token match, :head
             next unless match = scan(/.+/)
-            kind = :plain
+            encoder.text_token match, :plain
           elsif match = scan(/Added: /)
-            tokens << [:begin_line, line_kind = :head]
-            tokens << [match, :head]
+            encoder.begin_line line_kind = :head
+            encoder.text_token match, :head
             next unless match = scan(/.+/)
-            kind = :plain
+            encoder.text_token match, :plain
             state = :added
-          elsif match = scan(/\\ /)
-            tokens << [:begin_line, line_kind = :change]
-            tokens << [match, :change]
-            next unless match = scan(/.+/)
-            kind = :plain
+          elsif match = scan(/\\ .*/)
+            encoder.text_token match, :comment
           elsif match = scan(/@@(?>[^@\n]*)@@/)
+            content_scanner.state = :initial unless match?(/\n\+/)
+            content_scanner_entry_state = nil
             if check(/\n|$/)
-              tokens << [:begin_line, line_kind = :change]
+              encoder.begin_line line_kind = :change
             else
-              tokens << [:open, :change]
+              encoder.begin_group :change
             end
-            tokens << [match[0,2], :change]
-            tokens << [match[2...-2], :plain]
-            tokens << [match[-2,2], :change]
-            tokens << [:close, :change] unless line_kind
+            encoder.text_token match[0,2], :change
+            encoder.text_token match[2...-2], :plain
+            encoder.text_token match[-2,2], :change
+            encoder.end_group :change unless line_kind
             next unless match = scan(/.+/)
-            kind = :plain
+            if options[:highlight_code]
+              content_scanner.tokenize match, :tokens => encoder
+            else
+              encoder.text_token match, :plain
+            end
+            next
           elsif match = scan(/\+/)
-            tokens << [:begin_line, line_kind = :insert]
-            tokens << [match, :insert]
+            encoder.begin_line line_kind = :insert
+            encoder.text_token match, :insert
             next unless match = scan(/.+/)
-            kind = :plain
+            if options[:highlight_code]
+              content_scanner.tokenize match, :tokens => encoder
+            else
+              encoder.text_token match, :plain
+            end
+            next
           elsif match = scan(/-/)
-            tokens << [:begin_line, line_kind = :delete]
-            tokens << [match, :delete]
-            next unless match = scan(/.+/)
-            kind = :plain
-          elsif scan(/ .*/)
-            kind = :comment
-          elsif scan(/.+/)
-            tokens << [:begin_line, line_kind = :comment]
-            kind = :plain
+            deleted_lines += 1
+            encoder.begin_line line_kind = :delete
+            encoder.text_token match, :delete
+            if options[:inline_diff] && deleted_lines == 1 && check(/(?>.*)\n\+(?>.*)$(?!\n\+)/)
+              content_scanner_entry_state = content_scanner.state
+              skip(/(.*)\n\+(.*)$/)
+              head, deletion, insertion, tail = diff self[1], self[2]
+              pre, deleted, post = content_scanner.tokenize [head, deletion, tail], :tokens => Tokens.new
+              encoder.tokens pre
+              unless deleted.empty?
+                encoder.begin_group :eyecatcher
+                encoder.tokens deleted
+                encoder.end_group :eyecatcher
+              end
+              encoder.tokens post
+              encoder.end_line line_kind
+              encoder.text_token "\n", :space
+              encoder.begin_line line_kind = :insert
+              encoder.text_token '+', :insert
+              content_scanner.state = content_scanner_entry_state || :initial
+              pre, inserted, post = content_scanner.tokenize [head, insertion, tail], :tokens => Tokens.new
+              encoder.tokens pre
+              unless inserted.empty?
+                encoder.begin_group :eyecatcher
+                encoder.tokens inserted
+                encoder.end_group :eyecatcher
+              end
+              encoder.tokens post
+            elsif match = scan(/.*/)
+              if options[:highlight_code]
+                if deleted_lines == 1
+                  content_scanner_entry_state = content_scanner.state
+                end
+                content_scanner.tokenize match, :tokens => encoder unless match.empty?
+                if !match?(/\n-/)
+                  if match?(/\n\+/)
+                    content_scanner.state = content_scanner_entry_state || :initial
+                  end
+                  content_scanner_entry_state = nil
+                end
+              else
+                encoder.text_token match, :plain
+              end
+            end
+            next
+          elsif match = scan(/ .*/)
+            if options[:highlight_code]
+              content_scanner.tokenize match, :tokens => encoder
+            else
+              encoder.text_token match, :plain
+            end
+            next
+          elsif match = scan(/.+/)
+            encoder.begin_line line_kind = :comment
+            encoder.text_token match, :plain
           else
             raise_inspect 'else case rached'
           end
         
         when :added
           if match = scan(/   \+/)
-            tokens << [:begin_line, line_kind = :insert]
-            tokens << [match, :insert]
+            encoder.begin_line line_kind = :insert
+            encoder.text_token match, :insert
             next unless match = scan(/.+/)
-            kind = :plain
+            encoder.text_token match, :plain
           else
             state = :initial
             next
           end
         end
         
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
-        end
-        raise_inspect 'Empty token', tokens unless match
-        
-        tokens << [match, kind]
       end
       
-      tokens << [:end_line, line_kind] if line_kind
-      tokens
+      encoder.end_line line_kind if line_kind
+      
+      encoder
+    end
+    
+  private
+    
+    def diff a, b
+      # i will be the index of the leftmost difference from the left.
+      i_max = [a.size, b.size].min
+      i = 0
+      i += 1 while i < i_max && a[i] == b[i]
+      # j_min will be the index of the leftmost difference from the right.
+      j_min = i - i_max
+      # j will be the index of the rightmost difference from the right which
+      # does not precede the leftmost one from the left.
+      j = -1
+      j -= 1 while j >= j_min && a[j] == b[j]
+      return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j+1..-1] : ''
     end
     
   end