coderay 0.8.357 → 0.9.1

data/lib/coderay/encoders/lines_of_code.rb

@@ -0,0 +1,90 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+  
+  # Counts the LoC (Lines of Code). Returns an Integer >= 0.
+  # 
+  # Alias: :loc
+  # 
+  # Everything that is not comment, markup, doctype/shebang, or an empty line,
+  # is considered to be code.
+  # 
+  # For example,
+  # * HTML files not containing JavaScript have 0 LoC
+  # * in a Java class without comments, LoC is the number of non-empty lines
+  # 
+  # A Scanner class should define the token kinds that are not code in the
+  # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
+  class LinesOfCode < Encoder
+    
+    register_for :lines_of_code
+    
+    NON_EMPTY_LINE = /^\s*\S.*$/
+    
+    def compile tokens, options
+      if scanner = tokens.scanner
+        kinds_not_loc = scanner.class::KINDS_NOT_LOC
+      else
+        warn ArgumentError, 'Tokens have no scanner.' if $DEBUG
+        kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
+      end
+      code = tokens.token_class_filter :exclude => kinds_not_loc
+      @loc = code.text.scan(NON_EMPTY_LINE).size
+    end
+    
+    def finish options
+      @loc
+    end
+    
+  end
+  
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class LinesOfCodeTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :loc
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+    assert_nothing_raised do
+      filter = CodeRay.encoder :lines_of_code
+    end
+    assert_kind_of CodeRay::Encoders::LinesOfCode, filter
+  end
+  
+  def test_lines_of_code
+    tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+    assert_equal 1, tokens.lines_of_code
+    assert_equal 1, tokens.loc
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    tokens << ["Hello\n", :world]
+    tokens << ["Hello\n", :space]
+    tokens << ["Hello\n", :comment]
+    assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
+    assert_equal 2, tokens.lines_of_code
+    assert_equal 2, tokens.loc
+  end
+  
+end

data/lib/coderay/encoders/page.rb

@@ -9,11 +9,10 @@ module Encoders
 
     register_for :page
 
-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
       :css => :class,
       :wrap => :page,
       :line_numbers => :table
-    })
 
   end
 

data/lib/coderay/encoders/span.rb

@@ -9,10 +9,9 @@ module Encoders
 
     register_for :span
 
-    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
       :css => :style,
-      :wrap => :span,
-    })
+      :wrap => :span
 
   end
 

data/lib/coderay/encoders/term.rb

@@ -0,0 +1,137 @@
+# encoders/term.rb
+# By Rob Aldred (http://robaldred.co.uk)
+# Based on idea by Nathan Weizenbaum (http://nex-3.com)
+# MIT License (http://www.opensource.org/licenses/mit-license.php)
+#
+# A CodeRay encoder that outputs code highlighted for a color terminal.
+# Check out http://robaldred.co.uk
+
+module CodeRay
+  module Encoders
+    class Term < Encoder
+      register_for :term
+
+      TOKEN_COLORS = {
+        :attribute_name => '33',
+        :attribute_name_fat => '33',
+        :attribute_value => '31',
+        :attribute_value_fat => '31',
+        :bin => '1;35',
+        :char => {:self => '36', :delimiter => '34'},
+        :class => '1;35',
+        :class_variable => '36',
+        :color => '32',
+        :comment => '37',
+        :constant => ['34', '4'],
+        :definition => '1;32',
+        :directive => ['32', '4'],
+        :doc => '46',
+        :doc_string => ['31', '4'],
+        :entity => '33',
+        :error => ['1;33', '41'],
+        :exception => '1;31',
+        :float => '1;35',
+        :function => '1;34',
+        :global_variable => '42',
+        :hex => '1;36',
+        :include => '33',
+        :integer => '1;34',
+        :interpreted => '1;35',
+        :label => '1;4',
+        :local_variable => '33',
+        :oct => '1;35',
+        :operator_name => '1;29',
+        :pre_constant => '1;36',
+        :pre_type => '1;30',
+        :predefined => ['4', '1;34'],
+        :preprocessor => '36',
+        :regexp => {
+          :content => '31',
+          :delimiter => '1;29',
+          :modifier => '35',
+          :function => '1;29'
+        },
+        :reserved => '1;31',
+        :shell => {:self => '42', :content => '1;29'},
+        :string => '32',
+        :symbol => '1;32',
+        :tag => '34',
+        :tag_fat => '1;34',
+        :tag_special => ['34', '4'],
+        :type => '1;34',
+        :variable => '34'
+      }
+      TOKEN_COLORS[:procedure] = TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
+      TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+
+      protected
+
+      def setup(options)
+        @out = ''
+        @opened = [nil]
+        @subcolors = nil
+      end
+
+      def finish(options)
+        super
+      end
+    
+      def token text, type = :plain
+        case text
+      
+        when nil
+          # raise 'Token with nil as text was given: %p' % [[text, type]] 
+      
+        when String
+        
+          if color = (@subcolors || TOKEN_COLORS)[type]
+            color = color[:self] || return if Hash === color
+
+            @out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
+            @out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
+          else
+            @out << text
+          end
+      
+        # token groups, eg. strings
+        when :open
+          @opened[0] = type
+          if color = TOKEN_COLORS[type]
+            if Hash === color
+              @subcolors = color
+              @out << col(color[:self]) if color[:self]
+            else
+              @subcolors = {}
+              @out << col(color)
+            end
+          end
+          @opened << type
+        when :close
+          if @opened.empty?
+            # nothing to close
+          else
+            if (@subcolors || {})[:self]
+              @out << col(0)
+            end
+            @subcolors = nil
+            @opened.pop
+          end
+      
+        # whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
+        when :begin_line
+        
+        when :end_line        
+      
+        else
+          raise 'unknown token kind: %p' % [text]
+        end
+      end
+
+      private
+
+      def col(color)
+        Array(color).map { |c| "\e[#{c}m" }.join
+      end
+    end
+  end
+end

data/lib/coderay/encoders/text.rb

@@ -14,16 +14,16 @@ module Encoders
 
   protected
     def setup options
-      @out = ''
+      super
       @sep = options[:separator]
     end
 
-    def token text, kind
-      @out << text + @sep if text.is_a? ::String
+    def text_token text, kind
+      text + @sep
     end
 
     def finish options
-      @out.chomp @sep
+      super.chomp @sep
     end
 
   end

data/lib/coderay/encoders/token_class_filter.rb

@@ -0,0 +1,84 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  class TokenClassFilter < Filter
+
+    include Streamable
+    register_for :token_class_filter
+
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+
+  protected
+    def setup options
+      super
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  end
+
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokenClassFilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
+    assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :token_class_filter
+    end
+    assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    for i in 1..10
+      tokens << [i.to_s, :index]
+      tokens << [' ', :space] if i < 10
+    end
+    assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
+    assert_equal 10, tokens.token_class_filter(:exclude => :space).size
+    assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
+    assert_equal 9, tokens.token_class_filter(:include => :space).size
+    assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
+    assert_equal 0, tokens.token_class_filter(:exclude => :all).size
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens << [:open, :index]
+      tokens << [i.to_s, :content]
+      tokens << [:close, :index]
+    end
+    assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
+    assert_equal 20, tokens.token_class_filter(:include => :blubb).size
+    assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
+    assert_equal 30, tokens.token_class_filter(:exclude => :index).size
+  end
+  
+end

data/lib/coderay/encoders/xml.rb

@@ -29,6 +29,7 @@ module Encoders
     end
 
     def finish options
+      @out = ''
       @doc.write @out, options[:pretty], options[:transitive], true
       @out
     end

data/lib/coderay/for_redcloth.rb

@@ -29,21 +29,26 @@ module CodeRay
           }
           html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
         end
-        undef_method :code, :bc_open, :bc_close, :escape_pre
+        undef code, bc_open, bc_close, escape_pre
         def code(opts)  # :nodoc:
           opts[:block] = true
           if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
             # simulating pre-4.2 behavior
-            opts[:text].sub!(/\A\[(\w+)\]/, '')
-            opts[:lang] = $1
+            if opts[:text].sub!(/\A\[(\w+)\]/, '')
+              if CodeRay::Scanners[$1].plugin_id == 'plaintext'
+                opts[:text] = $& + opts[:text]
+              else
+                opts[:lang] = $1
+              end
+            end
           end
           if opts[:lang] && !filter_coderay
             require 'coderay'
             @in_bc ||= nil
             format = @in_bc ? :div : :span
+            opts[:text] = unescape(opts[:text]) unless @in_bc
             highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
             highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
-            highlighted_code = unescape(highlighted_code) unless @in_bc
             highlighted_code
           else
             "<code#{pba(opts)}>#{opts[:text]}</code>"

data/lib/coderay/helpers/file_type.rb

@@ -35,10 +35,12 @@ module FileType
     def [] filename, read_shebang = false
       name = File.basename filename
       ext = File.extname(name).sub(/^\./, '')  # from last dot, delete the leading dot
-      ext2 = filename[/\.(.*)/, 1]  # from first dot
+      ext2 = filename.to_s[/\.(.*)/, 1]  # from first dot
 
       type =
+        TypeFromExt[ext] ||
         TypeFromExt[ext.downcase] ||
+        (TypeFromExt[ext2] if ext2) ||
         (TypeFromExt[ext2.downcase] if ext2) ||
         TypeFromName[name] ||
         TypeFromName[name.downcase]
@@ -81,31 +83,46 @@ module FileType
   end
 
   TypeFromExt = {
-    'rb' => :ruby,
-    'rbw' => :ruby,
-    'rake' => :ruby,
-    'mab' => :ruby,
-    'cpp' => :c,
     'c' => :c,
+    'css' => :css,
+    'diff' => :diff,
+    'dpr' => :delphi,
+    'groovy' => :groovy,
+    'gvy' => :groovy,
     'h' => :c,
+    'htm' => :html,
+    'html' => :html,
+    'html.erb' => :rhtml,
     'java' => :java,
     'js' => :java_script,
     'json' => :json,
-    'diff' => :diff,
+    'mab' => :ruby,
+    'pas' => :delphi,
     'patch' => :diff,
-    'css' => :css,
-    'xml' => :xml,
-    'htm' => :html,
-    'html' => :html,
-    'xhtml' => :xhtml,
+    'php' => :php,
+    'php3' => :php,
+    'php4' => :php,
+    'php5' => :php,
+    'py' => :python,
+    'py3' => :python,
+    'pyw' => :python,
+    'rake' => :ruby,
     'raydebug' => :debug,
+    'rb' => :ruby,
+    'rbw' => :ruby,
     'rhtml' => :rhtml,
-    'html.erb' => :rhtml,
-    'ss' => :scheme,
+    'rxml' => :ruby,
     'sch' => :scheme,
+    'sql' => :sql,
+    'ss' => :scheme,
+    'xhtml' => :xhtml,
+    'xml' => :xml,
     'yaml' => :yaml,
     'yml' => :yaml,
   }
+  for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
+    TypeFromExt[cpp_alias] = :cpp
+  end
 
   TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
 
@@ -126,7 +143,7 @@ end
 __END__
 require 'test/unit'
 
-class TC_FileType < Test::Unit::TestCase
+class FileTypeTests < Test::Unit::TestCase
   
   include CodeRay
   
@@ -156,6 +173,8 @@ class TC_FileType < Test::Unit::TestCase
 
   def test_ruby
     assert_equal :ruby, FileType['test.rb']
+    assert_equal :ruby, FileType['test.java.rb']
+    assert_equal :java, FileType['test.rb.java']
     assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
     assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
     assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
@@ -175,6 +194,17 @@ class TC_FileType < Test::Unit::TestCase
     assert_not_equal :c, FileType['~/projects/blabla/c']
   end
 
+  def test_cpp
+    assert_equal :cpp, FileType['test.c++']
+    assert_equal :cpp, FileType['test.cxx']
+    assert_equal :cpp, FileType['test.hh']
+    assert_equal :cpp, FileType['test.hpp']
+    assert_equal :cpp, FileType['test.cu']
+    assert_equal :cpp, FileType['test.C']
+    assert_not_equal :cpp, FileType['test.c']
+    assert_not_equal :cpp, FileType['test.h']
+  end
+
   def test_html
     assert_equal :html, FileType['test.htm']
     assert_equal :xhtml, FileType['test.xhtml']
@@ -190,6 +220,15 @@ class TC_FileType < Test::Unit::TestCase
     assert_not_equal :yaml, FileType['YAML']
   end
 
+  def test_pathname
+    require 'pathname'
+    pn = Pathname.new 'test.rb'
+    assert_equal :ruby, FileType[pn]
+    dir = Pathname.new '/etc/var/blubb'
+    assert_equal :ruby, FileType[dir + pn]
+    assert_equal :cpp, FileType[dir + 'test.cpp']
+  end
+
   def test_no_shebang
     dir = './test'
     if File.directory? dir