coderay-beta 0.9.1

data/lib/coderay/encoders/null.rb

@@ -0,0 +1,26 @@
+module CodeRay
+module Encoders
+
+  # = Null Encoder
+  #
+  # Does nothing and returns an empty string.
+  class Null < Encoder
+
+    include Streamable
+    register_for :null
+
+    # Defined for faster processing
+    def to_proc
+      proc {}
+    end
+
+  protected
+
+    def token(*)
+      # do nothing
+    end
+
+  end
+
+end
+end

data/lib/coderay/encoders/page.rb

@@ -0,0 +1,20 @@
+module CodeRay
+module Encoders
+
+  load :html
+
+  class Page < HTML
+
+    FILE_EXTENSION = 'html'
+
+    register_for :page
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css => :class,
+      :wrap => :page,
+      :line_numbers => :table
+
+  end
+
+end
+end

data/lib/coderay/encoders/span.rb

@@ -0,0 +1,19 @@
+module CodeRay
+module Encoders
+
+  load :html
+
+  class Span < HTML
+
+    FILE_EXTENSION = 'span.html'
+
+    register_for :span
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css => :style,
+      :wrap => :span
+
+  end
+
+end
+end

data/lib/coderay/encoders/statistic.rb

@@ -0,0 +1,77 @@
+module CodeRay
+module Encoders
+
+  # Makes a statistic for the given tokens.
+  class Statistic < Encoder
+
+    include Streamable
+    register_for :stats, :statistic
+
+    attr_reader :type_stats, :real_token_count
+
+  protected
+
+    TypeStats = Struct.new :count, :size
+
+    def setup options
+      @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+      @real_token_count = 0
+    end
+
+    def generate tokens, options
+      @tokens = tokens
+      super
+    end
+
+    def text_token text, kind
+      @real_token_count += 1 unless kind == :space
+      @type_stats[kind].count += 1
+      @type_stats[kind].size += text.size
+      @type_stats['TOTAL'].size += text.size
+      @type_stats['TOTAL'].count += 1
+    end
+
+    # TODO Hierarchy handling
+    def block_token action, kind
+      @type_stats['TOTAL'].count += 1
+      @type_stats['open/close'].count += 1
+    end
+
+    STATS = <<-STATS
+
+Code Statistics
+
+Tokens            %8d
+  Non-Whitespace  %8d
+Bytes Total       %8d
+
+Token Types (%d):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+%s
+      STATS
+# space                    12007   33.81 %     1.7
+    TOKEN_TYPES_ROW = <<-TKR
+  %-20s  %8d  %6.2f %%   %5.1f
+      TKR
+
+    def finish options
+      all = @type_stats['TOTAL']
+      all_count, all_size = all.count, all.size
+      @type_stats.each do |type, stat|
+        stat.size /= stat.count.to_f
+      end
+      types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
+        TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+      end.join
+      STATS % [
+        all_count, @real_token_count, all_size,
+        @type_stats.delete_if { |k, v| k.is_a? String }.size,
+        types_stats
+      ]
+    end
+
+  end
+
+end
+end

data/lib/coderay/encoders/term.rb

@@ -0,0 +1,137 @@
+# encoders/term.rb
+# By Rob Aldred (http://robaldred.co.uk)
+# Based on idea by Nathan Weizenbaum (http://nex-3.com)
+# MIT License (http://www.opensource.org/licenses/mit-license.php)
+#
+# A CodeRay encoder that outputs code highlighted for a color terminal.
+# Check out http://robaldred.co.uk
+
+module CodeRay
+  module Encoders
+    class Term < Encoder
+      register_for :term
+
+      TOKEN_COLORS = {
+        :attribute_name => '33',
+        :attribute_name_fat => '33',
+        :attribute_value => '31',
+        :attribute_value_fat => '31',
+        :bin => '1;35',
+        :char => {:self => '36', :delimiter => '34'},
+        :class => '1;35',
+        :class_variable => '36',
+        :color => '32',
+        :comment => '37',
+        :constant => ['34', '4'],
+        :definition => '1;32',
+        :directive => ['32', '4'],
+        :doc => '46',
+        :doc_string => ['31', '4'],
+        :entity => '33',
+        :error => ['1;33', '41'],
+        :exception => '1;31',
+        :float => '1;35',
+        :function => '1;34',
+        :global_variable => '42',
+        :hex => '1;36',
+        :include => '33',
+        :integer => '1;34',
+        :interpreted => '1;35',
+        :label => '1;4',
+        :local_variable => '33',
+        :oct => '1;35',
+        :operator_name => '1;29',
+        :pre_constant => '1;36',
+        :pre_type => '1;30',
+        :predefined => ['4', '1;34'],
+        :preprocessor => '36',
+        :regexp => {
+          :content => '31',
+          :delimiter => '1;29',
+          :modifier => '35',
+          :function => '1;29'
+        },
+        :reserved => '1;31',
+        :shell => {:self => '42', :content => '1;29'},
+        :string => '32',
+        :symbol => '1;32',
+        :tag => '34',
+        :tag_fat => '1;34',
+        :tag_special => ['34', '4'],
+        :type => '1;34',
+        :variable => '34'
+      }
+      TOKEN_COLORS[:procedure] = TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
+      TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+
+      protected
+
+      def setup(options)
+        @out = ''
+        @opened = [nil]
+        @subcolors = nil
+      end
+
+      def finish(options)
+        super
+      end
+    
+      def token text, type = :plain
+        case text
+      
+        when nil
+          # raise 'Token with nil as text was given: %p' % [[text, type]] 
+      
+        when String
+        
+          if color = (@subcolors || TOKEN_COLORS)[type]
+            color = color[:self] || return if Hash === color
+
+            @out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
+            @out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
+          else
+            @out << text
+          end
+      
+        # token groups, eg. strings
+        when :open
+          @opened[0] = type
+          if color = TOKEN_COLORS[type]
+            if Hash === color
+              @subcolors = color
+              @out << col(color[:self]) if color[:self]
+            else
+              @subcolors = {}
+              @out << col(color)
+            end
+          end
+          @opened << type
+        when :close
+          if @opened.empty?
+            # nothing to close
+          else
+            if (@subcolors || {})[:self]
+              @out << col(0)
+            end
+            @subcolors = nil
+            @opened.pop
+          end
+      
+        # whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
+        when :begin_line
+        
+        when :end_line        
+      
+        else
+          raise 'unknown token kind: %p' % [text]
+        end
+      end
+
+      private
+
+      def col(color)
+        Array(color).map { |c| "\e[#{c}m" }.join
+      end
+    end
+  end
+end

data/lib/coderay/encoders/text.rb

@@ -0,0 +1,32 @@
+module CodeRay
+module Encoders
+
+  class Text < Encoder
+
+    include Streamable
+    register_for :text
+
+    FILE_EXTENSION = 'txt'
+
+    DEFAULT_OPTIONS = {
+      :separator => ''
+    }
+
+  protected
+    def setup options
+      super
+      @sep = options[:separator]
+    end
+
+    def text_token text, kind
+      text + @sep
+    end
+
+    def finish options
+      super.chomp @sep
+    end
+
+  end
+
+end
+end

data/lib/coderay/encoders/token_class_filter.rb

@@ -0,0 +1,84 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  class TokenClassFilter < Filter
+
+    include Streamable
+    register_for :token_class_filter
+
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+
+  protected
+    def setup options
+      super
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  end
+
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenClassFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_class_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens << [i.to_s, :index]
+      tokens << [' ', :space] if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
+    assert_equal 10, tokens.token_class_filter(:exclude => :space).size
+    assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
+    assert_equal 9, tokens.token_class_filter(:include => :space).size
+    assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
+    assert_equal 0, tokens.token_class_filter(:exclude => :all).size
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens << [:open, :index]
+      tokens << [i.to_s, :content]
+      tokens << [:close, :index]
+    end
+    assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
+    assert_equal 20, tokens.token_class_filter(:include => :blubb).size
+    assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
+    assert_equal 30, tokens.token_class_filter(:exclude => :index).size
+  end
+  
+end

data/lib/coderay/encoders/xml.rb

@@ -0,0 +1,71 @@
+module CodeRay
+module Encoders
+
+  # = XML Encoder
+  #
+  # Uses REXML. Very slow.
+  class XML < Encoder
+
+    include Streamable
+    register_for :xml
+
+    FILE_EXTENSION = 'xml'
+
+    require 'rexml/document'
+
+    DEFAULT_OPTIONS = {
+      :tab_width => 8,
+      :pretty => -1,
+      :transitive => false,
+    }
+
+  protected
+
+    def setup options
+      @doc = REXML::Document.new
+      @doc << REXML::XMLDecl.new
+      @tab_width = options[:tab_width]
+      @root = @node = @doc.add_element('coderay-tokens')
+    end
+
+    def finish options
+      @out = ''
+      @doc.write @out, options[:pretty], options[:transitive], true
+      @out
+    end
+    
+    def text_token text, kind
+      if kind == :space
+        token = @node
+      else
+        token = @node.add_element kind.to_s
+      end
+      text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+        case
+        when space
+          token << REXML::Text.new(space, true)
+        when tab
+          token << REXML::Text.new(tab, true)
+        when nl
+          token << REXML::Text.new(nl, true)
+        else
+          token << REXML::Text.new($&)
+        end
+      end
+    end
+
+    def open_token kind
+      @node = @node.add_element kind.to_s
+    end
+
+    def close_token kind
+      if @node == @root
+        raise 'no token to close!'
+      end
+      @node = @node.parent
+    end
+
+  end
+
+end
+end