coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay/encoders/null.rb

@@ -1,18 +1,25 @@
 module CodeRay
 module Encoders
-  
+
   # = Null Encoder
   #
   # Does nothing and returns an empty string.
   class Null < Encoder
-    
+
     register_for :null
-    
-    def text_token text, kind
+
+    # Defined for faster processing.
+    def to_proc
+      proc {}
+    end
+
+  protected
+
+    def token(*)
       # do nothing
     end
-    
+
   end
-  
+
 end
 end

data/lib/coderay/encoders/page.rb

@@ -1,6 +1,6 @@
 module CodeRay
 module Encoders
-  
+
   load :html
   
   # Wraps the output into a HTML page, using CSS classes and
@@ -8,17 +8,17 @@ module Encoders
   # 
   # See Encoders::HTML for available options.
   class Page < HTML
-    
+
     FILE_EXTENSION = 'html'
-    
+
     register_for :page
-    
+
     DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
-      :css          => :class,
-      :wrap         => :page,
+      :css => :class,
+      :wrap => :page,
       :line_numbers => :table
-    
+
   end
-  
+
 end
 end

data/lib/coderay/encoders/span.rb

@@ -1,23 +1,22 @@
 module CodeRay
 module Encoders
-  
+
   load :html
-  
+
   # Wraps HTML output into a SPAN element, using inline styles by default.
   # 
   # See Encoders::HTML for available options.
   class Span < HTML
-    
+
     FILE_EXTENSION = 'span.html'
-    
+
     register_for :span
-    
+
     DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
-      :css          => :style,
-      :wrap         => :span,
-      :line_numbers => false
-    
+      :css => :style,
+      :wrap => :span
+
   end
-  
+
 end
 end

data/lib/coderay/encoders/statistic.rb

@@ -1,26 +1,60 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
-  
+
   # Makes a statistic for the given tokens.
   # 
   # Alias: +stats+
   class Statistic < Encoder
-    
-    register_for :statistic
-    
+
+    register_for :stats, :statistic
+
     attr_reader :type_stats, :real_token_count  # :nodoc:
-    
+
     TypeStats = Struct.new :count, :size  # :nodoc:
-    
+
   protected
-    
+
     def setup options
-      super
-      
       @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
       @real_token_count = 0
     end
+
+    def generate tokens, options
+      @tokens = tokens
+      super
+    end
+
+    def text_token text, kind
+      @real_token_count += 1 unless kind == :space
+      @type_stats[kind].count += 1
+      @type_stats[kind].size += text.size
+      @type_stats['TOTAL'].size += text.size
+      @type_stats['TOTAL'].count += 1
+    end
+
+    # TODO Hierarchy handling
+    def begin_group kind
+      block_token 'begin_group'
+    end
+
+    def end_group kind
+      block_token 'end_group'
+    end
+
+    def begin_line kind
+      block_token 'begin_line'
+    end
+
+    def end_line kind
+      block_token 'end_line'
+    end
     
+    def block_token action
+      @type_stats['TOTAL'].count += 1
+      @type_stats[action].count += 1
+    end
+
     STATS = <<-STATS  # :nodoc:
 
 Code Statistics
@@ -33,12 +67,12 @@ Token Types (%d):
   type                     count     ratio    size (average)
 -------------------------------------------------------------
 %s
-    STATS
-    
+      STATS
+# space                    12007   33.81 %     1.7
     TOKEN_TYPES_ROW = <<-TKR  # :nodoc:
   %-20s  %8d  %6.2f %%   %5.1f
-    TKR
-    
+      TKR
+
     def finish options
       all = @type_stats['TOTAL']
       all_count, all_size = all.count, all.size
@@ -48,49 +82,78 @@ Token Types (%d):
       types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
         TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
       end.join
-      @out << STATS % [
+      STATS % [
         all_count, @real_token_count, all_size,
         @type_stats.delete_if { |k, v| k.is_a? String }.size,
         types_stats
       ]
-      
-      super
-    end
-    
-  public
-    
-    def text_token text, kind
-      @real_token_count += 1 unless kind == :space
-      @type_stats[kind].count += 1
-      @type_stats[kind].size += text.size
-      @type_stats['TOTAL'].size += text.size
-      @type_stats['TOTAL'].count += 1
-    end
-    
-    # TODO Hierarchy handling
-    def begin_group kind
-      block_token ':begin_group', kind
-    end
-    
-    def end_group kind
-      block_token ':end_group', kind
-    end
-    
-    def begin_line kind
-      block_token ':begin_line', kind
-    end
-    
-    def end_line kind
-      block_token ':end_line', kind
     end
-    
-    def block_token action, kind
-      @type_stats['TOTAL'].count += 1
-      @type_stats[action].count += 1
-      @type_stats[kind].count += 1
-    end
-    
+
   end
-  
+
+end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
 end
+
+__END__
+require 'test/unit'
+
+class StatisticEncoderTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder
+    stats = nil
+    assert_nothing_raised do
+      stats = CodeRay.encoder :statistic
+    end
+    assert_kind_of CodeRay::Encoders::Encoder, stats
+  end
+  
+  TEST_INPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n", :space],
+    ["\n  \t", :space],
+    ["   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  TEST_OUTPUT = <<-'DEBUG'
+
+Code Statistics
+
+Tokens                  11
+  Non-Whitespace         4
+Bytes Total             20
+
+Token Types (5):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                       11  100.00 %     1.8
+  space                        3   27.27 %     3.0
+  begin_group                  1    9.09 %     0.0
+  begin_line                   1    9.09 %     0.0
+  content                      1    9.09 %     4.0
+  end_group                    1    9.09 %     0.0
+  end_line                     1    9.09 %     0.0
+  integer                      1    9.09 %     2.0
+  method                       1    9.09 %     2.0
+  operator                     1    9.09 %     3.0
+
+  DEBUG
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT)
+    assert_equal TEST_OUTPUT, TEST_INPUT.statistic
+  end
+  
+end

data/lib/coderay/encoders/terminal.rb

@@ -22,7 +22,7 @@ module CodeRay
         :annotation => '35',
         :attribute_name => '33',
         :attribute_value => '31',
-        :binary => '1;35',
+        :bin => '1;35',
         :char => {
           :self => '36', :delimiter => '34'
         },
@@ -47,13 +47,14 @@ module CodeRay
         :hex => '1;36',
         :include => '33',
         :integer => '1;34',
+        :interpreted => '1;35',
         :key => '35',
         :label => '1;15',
         :local_variable => '33',
-        :octal => '1;35',
+        :oct => '1;35',
         :operator_name => '1;29',
-        :predefined_constant => '1;36',
-        :predefined_type => '1;30',
+        :pre_constant => '1;36',
+        :pre_type => '1;30',
         :predefined => ['4', '1;34'],
         :preprocessor => '36',
         :pseudo_class => '34',
@@ -78,6 +79,7 @@ module CodeRay
         },
         :symbol => '1;32',
         :tag => '34',
+        :tag_special => ['34', '4'],
         :type => '1;34',
         :value => '36',
         :variable => '34',
@@ -91,10 +93,11 @@ module CodeRay
       TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
       TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
       TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] =
-        TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
-      
+        TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] =
+        TOKEN_COLORS[:delimiter]
+
     protected
-      
+
       def setup(options)
         super
         @opened = []

data/lib/coderay/encoders/text.rb

@@ -1,10 +1,11 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
   # Concats the tokens into a single string, resulting in the original
   # code string if no tokens were removed.
   # 
-  # Alias: +plain+, +plaintext+
+  # Alias: +plain+
   # 
   # == Options
   # 
@@ -13,34 +14,52 @@ module Encoders
   # 
   # Default: empty String
   class Text < Encoder
-    
+
     register_for :text
-    
+
     FILE_EXTENSION = 'txt'
-    
+
     DEFAULT_OPTIONS = {
       :separator => nil
     }
-    
+
     def text_token text, kind
-      super
-      
-      if @first
-        @first = false
-      else
-        @out << @sep
-      end if @sep
+      @out << text
+      @out << @sep if @sep
     end
-    
+
   protected
     def setup options
       super
-      
-      @first = true
       @sep = options[:separator]
     end
-    
+
+    def finish options
+      super.chomp @sep
+    end
+
   end
-  
+
+end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
 end
+
+__END__
+require 'test/unit'
+
+class CountTest < Test::Unit::TestCase
+  
+  def test_count
+    ruby = <<-RUBY
+puts "Hello world!"
+    RUBY
+    tokens = CodeRay.scan ruby, :ruby
+    assert_equal ruby, tokens.encode_with(:text)
+  end
+  
+end

data/lib/coderay/encoders/token_kind_filter.rb

@@ -1,3 +1,4 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
@@ -34,7 +35,6 @@ module Encoders
   protected
     def setup options
       super
-      
       @group_excluded = false
       @exclude = options[:exclude]
       @exclude = Array(@exclude) unless @exclude == :all
@@ -109,3 +109,60 @@ module Encoders
   
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenKindFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_kind_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenKindFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens.text_token i.to_s, :index
+      tokens.text_token ' ', :space if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count
+    assert_equal 10, tokens.token_kind_filter(:exclude => :space).count
+    assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count
+    assert_equal 9, tokens.token_kind_filter(:include => :space).count
+    assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count
+    assert_equal 0, tokens.token_kind_filter(:exclude => :all).count
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
+      tokens.begin_group :naught if i == 5
+      tokens.end_group :naught if i == 7
+      tokens.begin_line :blubb
+      tokens.text_token i.to_s, :content
+      tokens.end_line :blubb
+    end
+    assert_equal 16, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count
+    assert_equal 16, tokens.token_kind_filter(:include => :blubb).count
+    assert_equal 24, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => [:blubb, :content]).count
+    assert_equal 24, tokens.token_kind_filter(:include => [:blubb, :content]).count
+    assert_equal 32, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count
+    assert_equal 32, tokens.token_kind_filter(:exclude => :index).count
+  end
+  
+end