coderay 1.0.0.738.pre → 1.0.0.778.pre

data/Rakefile

@@ -1,9 +1,8 @@
 $:.unshift File.dirname(__FILE__) unless $:.include? '.'
-require 'rake/rdoctask'
 
 ROOT = '.'
 LIB_ROOT = File.join ROOT, 'lib'
-EXTRA_RDOC_FILES = %w(lib/README FOLDERS)
+EXTRA_RDOC_FILES = %w(README.rdoc FOLDERS)
 
 task :default => :test
 
@@ -23,13 +22,14 @@ else
     ruby './test/functional/for_redcloth.rb'
   end
   
+  gem 'rdoc' if defined? gem
+  require 'rdoc/task'
   desc 'Generate documentation for CodeRay'
   Rake::RDocTask.new :doc do |rd|
     rd.title = 'CodeRay Documentation'
-    rd.main = 'lib/README'
+    rd.main = 'README.rdoc'
     rd.rdoc_files.add Dir['lib']
-    rd.rdoc_files.add 'lib/README'
-    rd.rdoc_files.add 'FOLDERS'
+    rd.rdoc_files.add 'README.rdoc'
     rd.rdoc_dir = 'doc'
   end
   

data/lib/coderay.rb

@@ -168,7 +168,7 @@ module CodeRay
     #
     # If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
     # determine it. If it cannot find out what type it is, it uses
-    # CodeRay::Scanners::Plaintext.
+    # CodeRay::Scanners::Text.
     #
     # Calls CodeRay.scan.
     #
@@ -179,7 +179,7 @@ module CodeRay
       file = IO.read filename
       if lang == :auto
         require 'coderay/helpers/file_type'
-        lang = FileType.fetch filename, :plaintext, true
+        lang = FileType.fetch filename, :text, true
       end
       scan file, lang, options = {}, &block
     end

data/lib/coderay/encoder.rb

@@ -46,7 +46,7 @@ module CodeRay
       DEFAULT_OPTIONS = { }
       
       # The options you gave the Encoder at creating.
-      attr_accessor :options
+      attr_accessor :options, :scanner
       
       # Creates a new Encoder.
       # +options+ is saved and used for all encode operations, as long
@@ -66,6 +66,7 @@ module CodeRay
       # Encode a Tokens object.
       def encode_tokens tokens, options = {}
         options = @options.merge options
+        @scanner = tokens.scanner if tokens.respond_to? :scanner
         setup options
         compile tokens, options
         finish options
@@ -74,10 +75,9 @@ module CodeRay
       # Encode the given +code+ using the Scanner for +lang+.
       def encode code, lang, options = {}
         options = @options.merge options
+        @scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
         setup options
-        scanner_options = CodeRay.get_scanner_options options
-        scanner_options[:tokens] = self
-        CodeRay.scan code, lang, scanner_options
+        @scanner.tokenize
         finish options
       end
       

data/lib/coderay/encoders/_map.rb

@@ -2,9 +2,11 @@ module CodeRay
 module Encoders
   
   map \
-    :loc => :lines_of_code,
+    :loc  => :lines_of_code,
     :term => :terminal,
+    :tty  => :terminal,
     :plain => :text,
+    :plaintext => :text,
     :remove_comments => :comment_filter,
     :stats => :statistic,
     :tex => :latex

data/lib/coderay/encoders/html.rb

@@ -136,13 +136,7 @@ module Encoders
     HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
     
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
-      h[kind] =
-        case kind
-        when :pre_constant  # FIXME: rename to :predefined_constant
-          'Predefined constant'
-        else
-          kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
-        end
+      h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
     end
     
     TRANSPARENT_TOKEN_KINDS = Set[
@@ -153,18 +147,14 @@ module Encoders
     #
     # +hint+ may be :info, :info_long or :debug.
     def self.token_path_to_hint hint, kinds
-      # FIXME: TRANSPARENT_TOKEN_KINDS?
-      # if TRANSPARENT_TOKEN_KINDS.include? kinds.first
-      #   kinds = kinds[1..-1]
-      # else
-      #   kinds = kinds[1..-1] + kinds.first
-      # end
+      kinds = Array kinds
       title =
         case hint
         when :info
-          TOKEN_KIND_TO_INFO[Array(kinds).first]
+          kinds = kinds[1..-1] if TRANSPARENT_TOKEN_KINDS.include? kinds.first
+          TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          kinds.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
           kinds.inspect
         end
@@ -184,7 +174,7 @@ module Encoders
       hint = options[:hint]
       if hint && ![:debug, :info, :info_long].include?(hint)
         raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :debug, false, or nil." % hint
+          expected :info, :info_long, :debug, false, or nil." % hint
       end
       
       css_classes = TokenKinds
@@ -218,6 +208,8 @@ module Encoders
       else
         raise ArgumentError, "Unknown value %p for :css." % options[:css]
       end
+      
+      @set_last_opened = options[:hint] || options[:css] == :style
     end
     
     def finish options
@@ -255,7 +247,7 @@ module Encoders
     def begin_group kind
       @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
       @opened << kind
-      @last_opened = kind if @options[:css] == :style
+      @last_opened = kind if @set_last_opened
     end
     
     def end_group kind

data/lib/coderay/encoders/html/numbering.rb

@@ -68,18 +68,18 @@ module Encoders
         when :inline
           max_width = (start + line_count).to_s.size
           line_number = start
-          opened_tags = []
+          nesting = []
           output.gsub!(/^.*$\n?/) do |line|
             line.chomp!
-            open = opened_tags.join
+            open = nesting.join
             line.scan(%r!<(/)?span[^>]*>?!) do |close,|
               if close
-                opened_tags.pop
+                nesting.pop
               else
-                opened_tags << $&
+                nesting << $&
               end
             end
-            close = '</span>' * opened_tags.size
+            close = '</span>' * nesting.size
             
             line_number_text = bolding.call line_number
             indent = ' ' * (max_width - line_number.to_s.size)  # TODO: Optimize (10^x)

data/lib/coderay/encoders/html/output.rb

@@ -3,14 +3,14 @@ module Encoders
 
   class HTML
 
-    # This module is included in the output String from thew HTML Encoder.
+    # This module is included in the output String of the HTML Encoder.
     #
     # It provides methods like wrap, div, page etc.
     #
     # Remember to use #clone instead of #dup to keep the modules the object was
     # extended with.
     #
-    # TODO: more doc.
+    # TODO: Rewrite this without monkey patching.
     module Output
 
       attr_accessor :css
@@ -25,7 +25,7 @@ module Encoders
 
         def make_stylesheet css, in_tag = false  # :nodoc:
           sheet = css.stylesheet
-          sheet = <<-CSS if in_tag
+          sheet = <<-'CSS' if in_tag
 <style type="text/css">
 #{sheet}
 </style>

data/lib/coderay/encoders/lines_of_code.rb

@@ -14,7 +14,7 @@ module Encoders
   # 
   # A Scanner class should define the token kinds that are not code in the
   # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
-  class LinesOfCode < Encoder
+  class LinesOfCode < TokenKindFilter
     
     register_for :lines_of_code
     
@@ -23,22 +23,19 @@ module Encoders
   protected
     
     def setup options
-      @out = 0
-    end
-    
-    def compile tokens, options
-      if scanner = tokens.scanner
+      if scanner
         kinds_not_loc = scanner.class::KINDS_NOT_LOC
       else
         warn "Tokens have no associated scanner, counting all nonempty lines." if $VERBOSE
         kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
       end
-      code = tokens.token_kind_filter :exclude => kinds_not_loc
-      @out = code.to_s.scan(NON_EMPTY_LINE).size
+      
+      options[:exclude] = kinds_not_loc
+      super options
     end
     
     def finish options
-      @out
+      @out.to_s.scan(NON_EMPTY_LINE).size
     end
     
   end

data/lib/coderay/encoders/statistic.rb

@@ -1,29 +1,29 @@
 module CodeRay
 module Encoders
-
+  
   # Makes a statistic for the given tokens.
   # 
   # Alias: +stats+
   class Statistic < Encoder
-
-    register_for :stats, :statistic
-
+    
+    register_for :statistic
+    
     attr_reader :type_stats, :real_token_count  # :nodoc:
-
+    
     TypeStats = Struct.new :count, :size  # :nodoc:
-
+    
   protected
-
+    
     def setup options
       @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
       @real_token_count = 0
     end
-
+    
     def generate tokens, options
       @tokens = tokens
       super
     end
-  
+    
     STATS = <<-STATS  # :nodoc:
 
 Code Statistics
@@ -36,12 +36,12 @@ Token Types (%d):
   type                     count     ratio    size (average)
 -------------------------------------------------------------
 %s
-      STATS
-# space                    12007   33.81 %     1.7
+    STATS
+    
     TOKEN_TYPES_ROW = <<-TKR  # :nodoc:
   %-20s  %8d  %6.2f %%   %5.1f
-      TKR
-
+    TKR
+    
     def finish options
       all = @type_stats['TOTAL']
       all_count, all_size = all.count, all.size
@@ -57,7 +57,7 @@ Token Types (%d):
         types_stats
       ]
     end
-
+    
   public
     
     def text_token text, kind
@@ -67,30 +67,31 @@ Token Types (%d):
       @type_stats['TOTAL'].size += text.size
       @type_stats['TOTAL'].count += 1
     end
-
+    
     # TODO Hierarchy handling
     def begin_group kind
-      block_token 'begin_group'
+      block_token ':begin_group', kind
     end
-
+    
     def end_group kind
-      block_token 'end_group'
+      block_token ':end_group', kind
     end
-
+    
     def begin_line kind
-      block_token 'begin_line'
+      block_token ':begin_line', kind
     end
-
+    
     def end_line kind
-      block_token 'end_line'
+      block_token ':end_line', kind
     end
     
-    def block_token action
+    def block_token action, kind
       @type_stats['TOTAL'].count += 1
       @type_stats[action].count += 1
+      @type_stats[kind].count += 1
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/terminal.rb

@@ -53,8 +53,8 @@ module CodeRay
         :local_variable => '33',
         :oct => '1;35',
         :operator_name => '1;29',
-        :pre_constant => '1;36',
-        :pre_type => '1;30',
+        :predefined_constant => '1;36',
+        :predefined_type => '1;30',
         :predefined => ['4', '1;34'],
         :preprocessor => '36',
         :pseudo_class => '34',

data/lib/coderay/encoders/text.rb

@@ -4,7 +4,7 @@ module Encoders
   # Concats the tokens into a single string, resulting in the original
   # code string if no tokens were removed.
   # 
-  # Alias: +plain+
+  # Alias: +plain+, +plaintext+
   # 
   # == Options
   # 
@@ -13,31 +13,31 @@ module Encoders
   # 
   # Default: empty String
   class Text < Encoder
-
+    
     register_for :text
-
+    
     FILE_EXTENSION = 'txt'
-
+    
     DEFAULT_OPTIONS = {
       :separator => nil
     }
-
+    
     def text_token text, kind
       super
       @out << @sep if @sep
     end
-
+    
   protected
     def setup options
       super
       @sep = options[:separator]
     end
-
+    
     def finish options
       super.chomp @sep
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/xml.rb

@@ -1,38 +1,38 @@
 module CodeRay
 module Encoders
-
+  
   # = XML Encoder
   #
   # Uses REXML. Very slow.
   class XML < Encoder
-
+    
     register_for :xml
-
+    
     FILE_EXTENSION = 'xml'
-
+    
     require 'rexml/document'
-
+    
     DEFAULT_OPTIONS = {
       :tab_width => 8,
       :pretty => -1,
       :transitive => false,
     }
-
+    
   protected
-
     def setup options
       @doc = REXML::Document.new
       @doc << REXML::XMLDecl.new
       @tab_width = options[:tab_width]
       @root = @node = @doc.add_element('coderay-tokens')
     end
-
+    
     def finish options
       @out = ''
       @doc.write @out, options[:pretty], options[:transitive], true
       @out
     end
     
+  public
     def text_token text, kind
       if kind == :space
         token = @node