coderay 0.9.8 → 1.0.0

data/lib/coderay/encoders/count.rb

@@ -1,21 +1,39 @@
 module CodeRay
 module Encoders
-
+  
+  # Returns the number of tokens.
+  # 
+  # Text and block tokens are counted.
   class Count < Encoder
-
-    include Streamable
+    
     register_for :count
-
-    protected
-
+    
+  protected
+    
     def setup options
-      @out = 0
+      super
+      
+      @count = 0
     end
-
-    def token text, kind
-      @out += 1
+    
+    def finish options
+      output @count
     end
+    
+  public
+    
+    def text_token text, kind
+      @count += 1
+    end
+    
+    def begin_group kind
+      @count += 1
+    end
+    alias end_group begin_group
+    alias begin_line begin_group
+    alias end_line begin_group
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/debug.rb

@@ -1,6 +1,6 @@
 module CodeRay
 module Encoders
-
+  
   # = Debug Encoder
   #
   # Fast encoder producing simple debug output.
@@ -10,40 +10,52 @@ module Encoders
   # You cannot fully restore the tokens information from the
   # output, because consecutive :space tokens are merged.
   # Use Tokens#dump for caching purposes.
+  # 
+  # See also: Scanners::Debug
   class Debug < Encoder
-
-    include Streamable
+    
     register_for :debug
-
+    
     FILE_EXTENSION = 'raydebug'
-
-  protected
+    
+    def initialize options = {}
+      super
+      @opened = []
+    end
+    
     def text_token text, kind
       if kind == :space
-        text
+        @out << text
       else
+        # TODO: Escape (
         text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
-        "#{kind}(#{text})"
+        @out << kind.to_s << '(' << text << ')'
       end
     end
-
-    def open_token kind
-      "#{kind}<"
+    
+    def begin_group kind
+      @opened << kind
+      @out << kind.to_s << '<'
     end
-
-    def close_token kind
-      ">"
+    
+    def end_group kind
+      if @opened.last != kind
+        puts @out
+        raise "we are inside #{@opened.inspect}, not #{kind}"
+      end
+      @opened.pop
+      @out << '>'
     end
-
+    
     def begin_line kind
-      "#{kind}["
+      @out << kind.to_s << '['
     end
-
+    
     def end_line kind
-      "]"
+      @out << ']'
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/div.rb

@@ -1,19 +1,23 @@
 module CodeRay
 module Encoders
-
+  
   load :html
-
+  
+  # Wraps HTML output into a DIV element, using inline styles by default.
+  # 
+  # See Encoders::HTML for available options.
   class Div < HTML
-
+    
     FILE_EXTENSION = 'div.html'
-
+    
     register_for :div
-
+    
     DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
-      :css => :style,
-      :wrap => :div
-
+      :css          => :style,
+      :wrap         => :div,
+      :line_numbers => false
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/filter.rb

@@ -1,75 +1,58 @@
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
+  # A Filter encoder has another Tokens instance as output.
+  # It can be subclass to select, remove, or modify tokens in the stream.
+  # 
+  # Subclasses of Filter are called "Filters" and can be chained.
+  # 
+  # == Options
+  # 
+  # === :tokens
+  # 
+  # The Tokens object which will receive the output.
+  # 
+  # Default: Tokens.new
+  # 
+  # See also: TokenKindFilter
   class Filter < Encoder
     
     register_for :filter
     
   protected
     def setup options
-      @out = Tokens.new
+      super
+      
+      @tokens = options[:tokens] || Tokens.new
     end
     
-    def text_token text, kind
-      [text, kind] if include_text_token? text, kind
+    def finish options
+      output @tokens
     end
     
-    def include_text_token? text, kind
-      true
-    end
+  public
     
-    def block_token action, kind
-      [action, kind] if include_block_token? action, kind
+    def text_token text, kind  # :nodoc:
+      @tokens.text_token text, kind
     end
     
-    def include_block_token? action, kind
-      true
+    def begin_group kind  # :nodoc:
+      @tokens.begin_group kind
     end
     
-  end
-  
-end
-end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  $: << File.join(File.dirname(__FILE__), '..')
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class FilterTest < Test::Unit::TestCase
-  
-  def test_creation
-    assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
-    filter = nil
-    assert_nothing_raised do
-      filter = CodeRay.encoder :filter
+    def begin_line kind  # :nodoc:
+      @tokens.begin_line kind
     end
-    assert_kind_of CodeRay::Encoders::Encoder, filter
-  end
-  
-  def test_filtering_text_tokens
-    tokens = CodeRay::Tokens.new
-    10.times do |i|
-      tokens << [i.to_s, :index]
+    
+    def end_group kind  # :nodoc:
+      @tokens.end_group kind
     end
-    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
-    assert_equal tokens, tokens.filter
-  end
-  
-  def test_filtering_block_tokens
-    tokens = CodeRay::Tokens.new
-    10.times do |i|
-      tokens << [:open, :index]
-      tokens << [i.to_s, :content]
-      tokens << [:close, :index]
+    
+    def end_line kind  # :nodoc:
+      @tokens.end_line kind
     end
-    assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
-    assert_equal tokens, tokens.filter
+    
   end
   
 end
+end

data/lib/coderay/encoders/html.rb

@@ -2,7 +2,7 @@ require 'set'
 
 module CodeRay
 module Encoders
-
+  
   # = HTML Encoder
   #
   # This is CodeRay's most important highlighter:
@@ -21,12 +21,12 @@ module Encoders
   #    :line_numbers => :inline,
   #    :css => :style
   #  )
-  #  #-> <span class="no">1</span>  <span style="color:#036; font-weight:bold;">Some</span> code
   #
   # == Options
   #
   # === :tab_width
   # Convert \t characters to +n+ spaces (a number.)
+  # 
   # Default: 8
   #
   # === :css
@@ -48,10 +48,18 @@ module Encoders
   # Default: 'CodeRay output'
   #
   # === :line_numbers
-  # Include line numbers in :table, :inline, :list or nil (no line numbers)
+  # Include line numbers in :table, :inline, or nil (no line numbers)
   #
   # Default: nil
   #
+  # === :line_number_anchors
+  # Adds anchors and links to the line numbers. Can be false (off), true (on),
+  # or a prefix string that will be prepended to the anchor name.
+  #
+  # The prefix must consist only of letters, digits, and underscores.
+  #
+  # Default: true, default prefix name: "line"
+  #
   # === :line_number_start
   # Where to start with line number counting.
   #
@@ -74,47 +82,48 @@ module Encoders
   #
   # === :hint
   # Include some information into the output using the title attribute.
-  # Can be :info (show token type on mouse-over), :info_long (with full path)
+  # Can be :info (show token kind on mouse-over), :info_long (with full path)
   # or :debug (via inspect).
   #
   # Default: false
   class HTML < Encoder
-
-    include Streamable
+    
     register_for :html
-
-    FILE_EXTENSION = 'html'
-
+    
+    FILE_EXTENSION = 'snippet.html'
+    
     DEFAULT_OPTIONS = {
       :tab_width => 8,
-
-      :css => :class,
-
-      :style => :cycnus,
-      :wrap => nil,
+      
+      :css   => :class,
+      :style => :alpha,
+      :wrap  => nil,
       :title => 'CodeRay output',
-
-      :line_numbers => nil,
-      :line_number_start => 1,
-      :bold_every => 10,
-      :highlight_lines => nil,
-
+      
+      :line_numbers        => nil,
+      :line_number_anchors => 'n',
+      :line_number_start   => 1,
+      :bold_every          => 10,
+      :highlight_lines     => nil,
+      
       :hint => false,
     }
-
-    helper :output, :css
-
+    
+    autoload :Output,    'coderay/encoders/html/output'
+    autoload :CSS,       'coderay/encoders/html/css'
+    autoload :Numbering, 'coderay/encoders/html/numbering'
+    
     attr_reader :css
-
+    
   protected
-
+    
     HTML_ESCAPE = {  #:nodoc:
       '&' => '&amp;',
       '"' => '&quot;',
       '>' => '&gt;',
       '<' => '&lt;',
     }
-
+    
     # This was to prevent illegal HTML.
     # Strange chars should still be avoided in codes.
     evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
@@ -124,185 +133,170 @@ module Encoders
     # \x9 (\t) and \xA (\n) not included
     #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
     HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
-
-    TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
-      h[kind] =
-        case kind
-        when :pre_constant
-          'Predefined constant'
-        else
-          kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
-        end
-    }
-
-    TRANSPARENT_TOKEN_KINDS = [
+    
+    TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
+      h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+    end
+    
+    TRANSPARENT_TOKEN_KINDS = Set[
       :delimiter, :modifier, :content, :escape, :inline_delimiter,
-    ].to_set
-
-    # Generate a hint about the given +classes+ in a +hint+ style.
+    ]
+    
+    # Generate a hint about the given +kinds+ in a +hint+ style.
     #
     # +hint+ may be :info, :info_long or :debug.
-    def self.token_path_to_hint hint, classes
+    def self.token_path_to_hint hint, kinds
+      kinds = Array kinds
       title =
         case hint
         when :info
-          TOKEN_KIND_TO_INFO[classes.first]
+          kinds = kinds[1..-1] if TRANSPARENT_TOKEN_KINDS.include? kinds.first
+          TOKEN_KIND_TO_INFO[kinds.first]
         when :info_long
-          classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
         when :debug
-          classes.inspect
+          kinds.inspect
         end
       title ? " title=\"#{title}\"" : ''
     end
-
+    
     def setup options
       super
-
+      
+      if options[:wrap] || options[:line_numbers]
+        @real_out = @out
+        @out = ''
+      end
+      
       @HTML_ESCAPE = HTML_ESCAPE.dup
       @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
-
-      @opened = [nil]
+      
+      @opened = []
+      @last_opened = nil
       @css = CSS.new options[:style]
-
+      
       hint = options[:hint]
-      if hint and not [:debug, :info, :info_long].include? hint
+      if hint && ![:debug, :info, :info_long].include?(hint)
         raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :debug, false, or nil." % hint
+          expected :info, :info_long, :debug, false, or nil." % hint
       end
-
+      
+      css_classes = TokenKinds
       case options[:css]
-
       when :class
-        @css_style = Hash.new do |h, k|
-          c = CodeRay::Tokens::ClassOfKind[k.first]
-          if c == :NO_HIGHLIGHT and not hint
-            h[k.dup] = false
-          else
-            title = if hint
-              HTML.token_path_to_hint(hint, k[1..-1] << k.first)
-            else
-              ''
-            end
-            if c == :NO_HIGHLIGHT
-              h[k.dup] = '<span%s>' % [title]
-            else
-              h[k.dup] = '<span%s class="%s">' % [title, c]
-            end
-          end
-        end
-
-      when :style
-        @css_style = Hash.new do |h, k|
-          if k.is_a? ::Array
-            styles = k.dup
+        @span_for_kind = Hash.new do |h, k|
+          if k.is_a? ::Symbol
+            kind = k_dup = k
           else
-            styles = [k]
+            kind = k.first
+            k_dup = k.dup
           end
-          classes = styles.map { |c| Tokens::ClassOfKind[c] }
-          if classes.first == :NO_HIGHLIGHT and not hint
-            h[k] = false
+          if kind != :space && (hint || css_class = css_classes[kind])
+            title = HTML.token_path_to_hint hint, k if hint
+            css_class ||= css_classes[kind]
+            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
           else
-            styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
-            title = HTML.token_path_to_hint hint, styles
-            style = @css[*classes]
-            h[k] =
-              if style
-                '<span%s style="%s">' % [title, style]
-              else
-                false
-              end
+            h[k_dup] = nil
           end
         end
-
+      when :style
+        @span_for_kind = Hash.new do |h, k|
+          kind = k.is_a?(Symbol) ? k : k.first
+          h[k.is_a?(Symbol) ? k : k.dup] =
+            if kind != :space && (hint || css_classes[kind])
+              title = HTML.token_path_to_hint hint, k if hint
+              style = @css.get_style Array(k).map { |c| css_classes[c] }
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            end
+        end
       else
         raise ArgumentError, "Unknown value %p for :css." % options[:css]
-
       end
+      
+      @set_last_opened = options[:hint] || options[:css] == :style
     end
-
+    
     def finish options
-      @opened.shift
-      @out << '</span>' * @opened.size
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened]
+        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
+        @out << '</span>' while @opened.pop
+        @last_opened = nil
       end
-
+      
       @out.extend Output
       @out.css = @css
-      @out.numerize! options[:line_numbers], options
+      if options[:line_numbers]
+        Numbering.number! @out, options[:line_numbers], options
+      end
       @out.wrap! options[:wrap]
       @out.apply_title! options[:title]
-
-      super
-    end
-
-    def token text, type = :plain
-      case text
       
-      when nil
-        # raise 'Token with nil as text was given: %p' % [[text, type]] 
-      
-      when String
-        if text =~ /#{HTML_ESCAPE_PATTERN}/o
-          text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
-        end
-        @opened[0] = type
-        if text != "\n" && style = @css_style[@opened]
-          @out << style << text << '</span>'
-        else
-          @out << text
-        end
-        
-      
-      # token groups, eg. strings
-      when :open
-        @opened[0] = type
-        @out << (@css_style[@opened] || '<span>')
-        @opened << type
-      when :close
-        if @opened.empty?
-          # nothing to close
-        else
-          if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a token (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
-          @out << '</span>'
-          @opened.pop
-        end
+      if defined?(@real_out) && @real_out
+        @real_out << @out
+        @out = @real_out
+      end
       
-      # whole lines to be highlighted, eg. a deleted line in a diff
-      when :begin_line
-        @opened[0] = type
-        if style = @css_style[@opened]
-          if style['class="']
-            @out << style.sub('class="', 'class="line ')
-          else
-            @out << style.sub('>', ' class="line">')
-          end
-        else
-          @out << '<span class="line">'
-        end
-        @opened << type
-      when :end_line
-        if @opened.empty?
-          # nothing to close
+      super
+    end
+    
+  public
+    
+    def text_token text, kind
+      if text =~ /#{HTML_ESCAPE_PATTERN}/o
+        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+      end
+      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+        @out << style << text << '</span>'
+      else
+        @out << text
+      end
+    end
+    
+    # token groups, eg. strings
+    def begin_group kind
+      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @opened << kind
+      @last_opened = kind if @set_last_opened
+    end
+    
+    def end_group kind
+      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
+        warn 'Malformed token stream: Trying to close a token (%p) ' \
+          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      end
+      if @opened.pop
+        @out << '</span>'
+        @last_opened = @opened.last if @last_opened
+      end
+    end
+    
+    # whole lines to be highlighted, eg. a deleted line in a diff
+    def begin_line kind
+      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+        if style['class="']
+          @out << style.sub('class="', 'class="line ')
         else
-          if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
-            raise 'Malformed token stream: Trying to close a line (%p) \
-              that is not open. Open are: %p.' % [type, @opened[1..-1]]
-          end
-          @out << '</span>'
-          @opened.pop
+          @out << style.sub('>', ' class="line">')
         end
-      
       else
-        raise 'unknown token kind: %p' % [text]
-        
+        @out << '<span class="line">'
       end
+      @opened << kind
+      @last_opened = kind if @options[:css] == :style
     end
-
+    
+    def end_line kind
+      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
+        warn 'Malformed token stream: Trying to close a line (%p) ' \
+          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      end
+      if @opened.pop
+        @out << '</span>'
+        @last_opened = @opened.last if @last_opened
+      end
+    end
+    
   end
-
+  
 end
 end