coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay/encoders/_map.rb

@@ -1,17 +1,13 @@
 module CodeRay
 module Encoders
-  
+
   map \
-    :loc             => :lines_of_code,
-    :plain           => :text,
-    :plaintext       => :text,
+    :loc => :lines_of_code,
+    :term => :terminal,
+    :plain => :text,
     :remove_comments => :comment_filter,
-    :stats           => :statistic,
-    :term            => :terminal,
-    :tty             => :terminal,
-    :yml             => :yaml
-  
-  # No default because Tokens#nonsense should raise NoMethodError.
-  
+    :stats => :statistic,
+    :tex => :latex
+
 end
 end

data/lib/coderay/encoders/comment_filter.rb

@@ -1,3 +1,4 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
@@ -23,3 +24,63 @@ module Encoders
   
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class CommentFilterTest < Test::Unit::TestCase
+  
+  def test_filtering_comments
+    tokens = CodeRay.scan <<-RUBY, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
+#!/usr/bin/env ruby
+
+puts "Hello world!"
+    RUBY_FILTERED
+  end
+  
+  def test_filtering_docstrings
+    tokens = CodeRay.scan <<-PYTHON, :python
+'''
+Assuming this is file mymodule.py then this string, being the
+first statement in the file will become the mymodule modules
+docstring when the file is imported
+'''
+
+class Myclass():
+    """
+    The class's docstring
+    """
+
+    def mymethod(self):
+        '''The method's docstring'''
+
+def myfunction():
+    """The function's docstring"""
+    PYTHON
+    assert_equal <<-PYTHON_FILTERED.chomp, tokens.comment_filter.text
+
+
+class Myclass():
+    
+
+    def mymethod(self):
+        
+
+def myfunction():
+    
+
+PYTHON_FILTERED
+  end
+  
+end

data/lib/coderay/encoders/count.rb

@@ -1,3 +1,4 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
@@ -11,23 +12,15 @@ module Encoders
   protected
     
     def setup options
-      super
-      
-      @count = 0
+      @out = 0
     end
     
-    def finish options
-      output @count
-    end
-    
-  public
-    
     def text_token text, kind
-      @count += 1
+      @out += 1
     end
     
     def begin_group kind
-      @count += 1
+      @out += 1
     end
     alias end_group begin_group
     alias begin_line begin_group
@@ -37,3 +30,25 @@ module Encoders
   
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class CountTest < Test::Unit::TestCase
+  
+  def test_count
+    tokens = CodeRay.scan <<-RUBY.strip, :ruby
+#!/usr/bin/env ruby
+# a minimal Ruby program
+puts "Hello world!"
+    RUBY
+    assert_equal 9, tokens.encode_with(:count)
+  end
+  
+end

data/lib/coderay/encoders/debug.rb

@@ -1,6 +1,7 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
-  
+
   # = Debug Encoder
   #
   # Fast encoder producing simple debug output.
@@ -13,31 +14,32 @@ module Encoders
   # 
   # See also: Scanners::Debug
   class Debug < Encoder
-    
+
     register_for :debug
-    
+
     FILE_EXTENSION = 'raydebug'
     
     def initialize options = {}
       super
       @opened = []
     end
-    
+
+  public
+  
     def text_token text, kind
       if kind == :space
         @out << text
       else
-        # TODO: Escape (
         text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
         @out << kind.to_s << '(' << text << ')'
       end
     end
-    
+
     def begin_group kind
       @opened << kind
       @out << kind.to_s << '<'
     end
-    
+
     def end_group kind
       if @opened.last != kind
         puts @out
@@ -46,16 +48,63 @@ module Encoders
       @opened.pop
       @out << '>'
     end
-    
+
     def begin_line kind
       @out << kind.to_s << '['
     end
-    
+
     def end_line kind
       @out << ']'
     end
-    
+
   end
-  
+
+end
+end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
 end
+
+__END__
+require 'test/unit'
+
+class DebugEncoderTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::Debug < CodeRay::Encoders::Encoder
+    debug = nil
+    assert_nothing_raised do
+      debug = CodeRay.encoder :debug
+    end
+    assert_kind_of CodeRay::Encoders::Encoder, debug
+  end
+  
+  TEST_INPUT = CodeRay::Tokens[
+    ['10', :integer],
+    ['(\\)', :operator],
+    [:begin_group, :string],
+    ['test', :content],
+    [:end_group, :string],
+    [:begin_line, :test],
+    ["\n", :space],
+    ["\n  \t", :space],
+    ["   \n", :space],
+    ["[]", :method],
+    [:end_line, :test],
+  ].flatten
+  TEST_OUTPUT = <<-'DEBUG'.chomp
+integer(10)operator((\\\))string<content(test)>test[
+
+  	   
+method([])]
+  DEBUG
+  
+  def test_filtering_text_tokens
+    assert_equal TEST_OUTPUT, CodeRay::Encoders::Debug.new.encode_tokens(TEST_INPUT)
+    assert_equal TEST_OUTPUT, TEST_INPUT.debug
+  end
+  
 end

data/lib/coderay/encoders/div.rb

@@ -1,23 +1,22 @@
 module CodeRay
 module Encoders
-  
+
   load :html
   
   # Wraps HTML output into a DIV element, using inline styles by default.
   # 
   # See Encoders::HTML for available options.
   class Div < HTML
-    
+
     FILE_EXTENSION = 'div.html'
-    
+
     register_for :div
-    
+
     DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
-      :css          => :style,
-      :wrap         => :div,
-      :line_numbers => false
-    
+      :css => :style,
+      :wrap => :div
+
   end
-  
+
 end
 end

data/lib/coderay/encoders/filter.rb

@@ -1,3 +1,4 @@
+($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
@@ -21,38 +22,77 @@ module Encoders
     
   protected
     def setup options
-      super
-      
-      @tokens = options[:tokens] || Tokens.new
-    end
-    
-    def finish options
-      output @tokens
+      @out = options[:tokens] || Tokens.new
     end
     
   public
     
     def text_token text, kind  # :nodoc:
-      @tokens.text_token text, kind
+      @out.text_token text, kind
     end
     
     def begin_group kind  # :nodoc:
-      @tokens.begin_group kind
+      @out.begin_group kind
     end
     
     def begin_line kind  # :nodoc:
-      @tokens.begin_line kind
+      @out.begin_line kind
     end
     
     def end_group kind  # :nodoc:
-      @tokens.end_group kind
+      @out.end_group kind
     end
     
     def end_line kind  # :nodoc:
-      @tokens.end_line kind
+      @out.end_line kind
     end
     
   end
   
 end
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class FilterTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
+    filter = nil
+    assert_nothing_raised do
+      filter = CodeRay.encoder :filter
+    end
+    assert_kind_of CodeRay::Encoders::Encoder, filter
+  end
+  
+  def test_filtering_text_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.text_token i.to_s, :index
+    end
+    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+    assert_equal tokens, tokens.filter
+  end
+  
+  def test_filtering_block_tokens
+    tokens = CodeRay::Tokens.new
+    10.times do |i|
+      tokens.begin_group :index
+      tokens.text_token i.to_s, :content
+      tokens.end_group :index
+      tokens.begin_line :index
+      tokens.text_token i.to_s, :content
+      tokens.end_line :index
+    end
+    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
+    assert_equal tokens, tokens.filter
+  end
+  
+end

data/lib/coderay/encoders/html.rb

@@ -2,7 +2,7 @@ require 'set'
 
 module CodeRay
 module Encoders
-  
+
   # = HTML Encoder
   #
   # This is CodeRay's most important highlighter:
@@ -21,6 +21,7 @@ module Encoders
   #    :line_numbers => :inline,
   #    :css => :style
   #  )
+  #  #-> <span class="no">1</span>  <span style="color:#036; font-weight:bold;">Some</span> code
   #
   # == Options
   #
@@ -87,43 +88,42 @@ module Encoders
   #
   # Default: false
   class HTML < Encoder
-    
+
     register_for :html
-    
-    FILE_EXTENSION = 'snippet.html'
-    
+
+    FILE_EXTENSION = 'html'
+
     DEFAULT_OPTIONS = {
       :tab_width => 8,
-      
-      :css   => :class,
+
+      :css => :class,
+
       :style => :alpha,
-      :wrap  => nil,
+      :wrap => nil,
       :title => 'CodeRay output',
-      
-      :line_numbers        => nil,
+
+      :line_numbers => nil,
       :line_number_anchors => 'n',
-      :line_number_start   => 1,
-      :bold_every          => 10,
-      :highlight_lines     => nil,
-      
+      :line_number_start => 1,
+      :bold_every => 10,
+      :highlight_lines => nil,
+
       :hint => false,
     }
-    
-    autoload :Output,    'coderay/encoders/html/output'
-    autoload :CSS,       'coderay/encoders/html/css'
-    autoload :Numbering, 'coderay/encoders/html/numbering'
-    
+
+    helper :output, :numbering, :css
+
     attr_reader :css
-    
+
   protected
-    
+
     HTML_ESCAPE = {  #:nodoc:
       '&' => '&amp;',
       '"' => '&quot;',
       '>' => '&gt;',
       '<' => '&lt;',
     }
-    
+
     # This was to prevent illegal HTML.
     # Strange chars should still be avoided in codes.
     evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
@@ -133,109 +133,110 @@ module Encoders
     # \x9 (\t) and \xA (\n) not included
     #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
     HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
-    
+
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
-      h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+      h[kind] =
+        case kind
+        when :pre_constant
+          'Predefined constant'
+        else
+          kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+        end
     end
-    
-    TRANSPARENT_TOKEN_KINDS = Set[
+
+    TRANSPARENT_TOKEN_KINDS = [
       :delimiter, :modifier, :content, :escape, :inline_delimiter,
-    ]
-    
+    ].to_set
+
     # Generate a hint about the given +kinds+ in a +hint+ style.
     #
     # +hint+ may be :info, :info_long or :debug.
     def self.token_path_to_hint hint, kinds
-      kinds = Array kinds
+      # FIXME: TRANSPARENT_TOKEN_KINDS?
+      # if TRANSPARENT_TOKEN_KINDS.include? kinds.first
+      #   kinds = kinds[1..-1]
+      # else
+      #   kinds = kinds[1..-1] + kinds.first
+      # end
       title =
         case hint
         when :info
-          kinds = kinds[1..-1] if TRANSPARENT_TOKEN_KINDS.include? kinds.first
           TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
           kinds.inspect
         end
       title ? " title=\"#{title}\"" : ''
     end
-    
+
     def setup options
       super
       
-      if options[:wrap] || options[:line_numbers]
-        @real_out = @out
-        @out = ''
-      end
-      
       @HTML_ESCAPE = HTML_ESCAPE.dup
       @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
       
-      @opened = []
-      @last_opened = nil
+      @opened = [nil]
       @css = CSS.new options[:style]
       
       hint = options[:hint]
-      if hint && ![:debug, :info, :info_long].include?(hint)
+      if hint and not [:debug, :info, :info_long].include? hint
         raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :info_long, :debug, false, or nil." % hint
+          expected :info, :debug, false, or nil." % hint
       end
-      
-      css_classes = TokenKinds
+
       case options[:css]
+
       when :class
-        @span_for_kind = Hash.new do |h, k|
-          if k.is_a? ::Symbol
-            kind = k_dup = k
-          else
-            kind = k.first
-            k_dup = k.dup
-          end
-          if kind != :space && (hint || css_class = css_classes[kind])
-            title = HTML.token_path_to_hint hint, k if hint
-            css_class ||= css_classes[kind]
-            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-          else
-            h[k_dup] = nil
-          end
+        @css_style = Hash.new do |h, k|
+          c = Tokens::AbbreviationForKind[k.first]
+          h[k.dup] = 
+            if c != :NO_HIGHLIGHT or (hint && k.first != :space)
+              if hint
+                title = HTML.token_path_to_hint hint, k
+              end
+              if c == :NO_HIGHLIGHT
+                '<span%s>' % [title]
+              else
+                '<span%s class="%s">' % [title, c]
+              end
+            end
         end
+
       when :style
-        @span_for_kind = Hash.new do |h, k|
-          kind = k.is_a?(Symbol) ? k : k.first
-          h[k.is_a?(Symbol) ? k : k.dup] =
-            if kind != :space && (hint || css_classes[kind])
-              title = HTML.token_path_to_hint hint, k if hint
-              style = @css.get_style Array(k).map { |c| css_classes[c] }
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
+        @css_style = Hash.new do |h, k|
+          classes = k.map { |c| Tokens::AbbreviationForKind[c] }
+          h[k.dup] =
+            if classes.first != :NO_HIGHLIGHT or (hint && k.first != :space)
+              if hint
+                title = HTML.token_path_to_hint hint, k
+              end
+              style = @css[*classes]
+              if style
+                '<span%s style="%s">' % [title, style]
+              end
             end
         end
+
       else
         raise ArgumentError, "Unknown value %p for :css." % options[:css]
+
       end
-      
-      @set_last_opened = options[:hint] || options[:css] == :style
     end
-    
+
     def finish options
+      not_needed = @opened.shift
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
-        @out << '</span>' while @opened.pop
-        @last_opened = nil
+        warn '%d tokens still open: %p' % [@opened.size, @opened]
+        @out << '</span>' * @opened.size
       end
       
       @out.extend Output
       @out.css = @css
-      if options[:line_numbers]
-        Numbering.number! @out, options[:line_numbers], options
-      end
+      @out.number! options[:line_numbers], options
       @out.wrap! options[:wrap]
       @out.apply_title! options[:title]
       
-      if defined?(@real_out) && @real_out
-        @real_out << @out
-        @out = @real_out
-      end
-      
       super
     end
     
@@ -245,58 +246,64 @@ module Encoders
       if text =~ /#{HTML_ESCAPE_PATTERN}/o
         text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
       end
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
-        @out << style << text << '</span>'
-      else
-        @out << text
-      end
+      @opened[0] = kind
+      @out <<
+        if style = @css_style[@opened]
+          style + text + '</span>'
+        else
+          text
+        end
     end
     
     # token groups, eg. strings
     def begin_group kind
-      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @opened[0] = kind
       @opened << kind
-      @last_opened = kind if @set_last_opened
+      @out << (@css_style[@opened] || '<span>')
     end
     
     def end_group kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
+      if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind)
         warn 'Malformed token stream: Trying to close a token (%p) ' \
           'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
       end
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      @out << 
+        if @opened.empty?
+          '' # nothing to close
+        else
+          @opened.pop
+          '</span>'
+        end
     end
     
     # whole lines to be highlighted, eg. a deleted line in a diff
     def begin_line kind
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
-        if style['class="']
-          @out << style.sub('class="', 'class="line ')
+      @opened[0] = kind
+      style = @css_style[@opened]
+      @opened << kind
+      @out <<
+        if style
+          style.sub '<span', '<div'
         else
-          @out << style.sub('>', ' class="line">')
+          '<div>'
         end
-      else
-        @out << '<span class="line">'
-      end
-      @opened << kind
-      @last_opened = kind if @options[:css] == :style
     end
     
     def end_line kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
+      if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != kind)
         warn 'Malformed token stream: Trying to close a line (%p) ' \
           'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
       end
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      @out <<
+        if @opened.empty?
+          ''  # nothing to close
+        else
+          @opened.pop
+          '</div>'
+        end
     end
-    
+
   end
-  
+
 end
 end