coderay 1.0.9 → 1.1.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 88c4f8c3def05dd733e1e3b7707fb7e1f2d20d54
+  data.tar.gz: fe1abf58ddac324444cf7659d9d5d335bf470b46
+SHA512:
+  metadata.gz: fad90cb9e8943b9cf55c8d54a5ab099d1012dcbdfa5208d58abb74aa6111d3d6d86cdb1f0f937b46533d813b59aa9ed1673942ab184618110349f38314117f2e
+  data.tar.gz: 1a2f782f48e427d181f557fded51236567200dd9b10a33a8668da98a5cccdb52872e4bbf0c68f2490a685b01430157bfb471ae892075cd25b56bb52e7fe7b56c

data/Rakefile

@@ -1,3 +1,5 @@
+require 'bundler/gem_tasks'
+
 $:.unshift File.dirname(__FILE__) unless $:.include? '.'
 
 ROOT = '.'

data/bin/coderay

@@ -125,7 +125,7 @@ when 'highlight', nil
     end
     
     if output_file
-      output_format ||= CodeRay::FileType[output_file]
+      output_format ||= CodeRay::FileType[output_file] || :plain
     else
       output_format ||= :terminal
     end
@@ -143,7 +143,6 @@ when 'highlight', nil
         if output_file
           File.open output_file, 'w'
         else
-          $stdout.sync = true
           $stdout
         end
       CodeRay.encode(input, input_lang, output_format, :out => file)
@@ -156,8 +155,9 @@ when 'highlight', nil
         puts boom.message
       end
       # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
-    rescue CodeRay::Scanners::Scanner::ScanError  # FIXME: rescue Errno::EPIPE
-      # this is sometimes raised by pagers; ignore [TODO: wtf?]
+    rescue CodeRay::Scanners::Scanner::ScanError
+      # this is sometimes raised by pagers; ignore
+    # FIXME: rescue Errno::EPIPE
     ensure
       file.close if output_file
     end

data/lib/coderay.rb

@@ -127,14 +127,14 @@ module CodeRay
   
   $CODERAY_DEBUG ||= false
   
-  CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay'
+  CODERAY_PATH = File.expand_path('../coderay', __FILE__)
   
   # Assuming the path is a subpath of lib/coderay/
   def self.coderay_path *path
     File.join CODERAY_PATH, *path
   end
   
-  require coderay_path('version')
+  require 'coderay/version'
   
   # helpers
   autoload :FileType,    coderay_path('helpers', 'file_type')
@@ -166,7 +166,6 @@ module CodeRay
     #
     # See also demo/demo_simple.
     def scan code, lang, options = {}, &block
-      # FIXME: return a proxy for direct-stream encoding
       TokensProxy.new code, lang, options, block
     end
     

data/lib/coderay/encoders/debug.rb

@@ -9,7 +9,6 @@ module Encoders
   #
   # You cannot fully restore the tokens information from the
   # output, because consecutive :space tokens are merged.
-  # Use Tokens#dump for caching purposes.
   # 
   # See also: Scanners::Debug
   class Debug < Encoder
@@ -18,37 +17,26 @@ module Encoders
     
     FILE_EXTENSION = 'raydebug'
     
-    def initialize options = {}
-      super
-      @opened = []
-    end
-    
     def text_token text, kind
       if kind == :space
         @out << text
       else
-        # TODO: Escape (
-        text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
-        @out << kind.to_s << '(' << text << ')'
+        text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
+        text = text.gsub(')',  '\\\\)')    if text.index(')')
+        @out << "#{kind}(#{text})"
       end
     end
     
     def begin_group kind
-      @opened << kind
-      @out << kind.to_s << '<'
+      @out << "#{kind}<"
     end
     
     def end_group kind
-      if @opened.last != kind
-        puts @out
-        raise "we are inside #{@opened.inspect}, not #{kind}"
-      end
-      @opened.pop
       @out << '>'
     end
     
     def begin_line kind
-      @out << kind.to_s << '['
+      @out << "#{kind}["
     end
     
     def end_line kind

data/lib/coderay/encoders/debug_lint.rb

@@ -0,0 +1,62 @@
+module CodeRay
+module Encoders
+  
+  load :lint
+  
+  # = Debug Lint Encoder
+  #
+  # Debug encoder with additional checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::Debug
+  class DebugLint < Debug
+    
+    register_for :debug_lint
+    
+    def text_token text, kind
+      raise Lint::EmptyToken, 'empty token' if text.empty?
+      super
+    end
+    
+    def begin_group kind
+      @opened << kind
+      super
+    end
+    
+    def end_group kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+      super
+    end
+    
+    def begin_line kind
+      @opened << kind
+      super
+    end
+    
+    def end_line kind
+      raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+      super
+    end
+    
+    protected
+    
+    def setup options
+      super
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
+      super
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/encoders/html.rb

@@ -126,22 +126,21 @@ module Encoders
     
   protected
     
-    HTML_ESCAPE = {  #:nodoc:
-      '&' => '&',
-      '"' => '"',
-      '>' => '>',
-      '<' => '&lt;',
-    }
+    def self.make_html_escape_hash
+      {
+        '&' => '&amp;',
+        '"' => '&quot;',
+        '>' => '&gt;',
+        '<' => '&lt;',
+        # "\t" => will be set to ' ' * options[:tab_width] during setup
+      }.tap do |hash|
+        # Escape ASCII control codes except \x9 == \t and \xA == \n.
+        (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
+      end
+    end
     
-    # This was to prevent illegal HTML.
-    # Strange chars should still be avoided in codes.
-    evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
-    evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
-    #ansi_chars = Array(0x7f..0xff)
-    #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
-    # \x9 (\t) and \xA (\n) not included
-    #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
-    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
+    HTML_ESCAPE = make_html_escape_hash
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
     
     TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
       h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -172,66 +171,28 @@ module Encoders
     def setup options
       super
       
+      check_options! options
+      
       if options[:wrap] || options[:line_numbers]
         @real_out = @out
         @out = ''
       end
       
-      options[:break_lines] = true if options[:line_numbers] == :inline
-      
       @break_lines = (options[:break_lines] == true)
       
-      @HTML_ESCAPE = HTML_ESCAPE.dup
-      @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
       
       @opened = []
       @last_opened = nil
       @css = CSS.new options[:style]
       
-      hint = options[:hint]
-      if hint && ![:debug, :info, :info_long].include?(hint)
-        raise ArgumentError, "Unknown value %p for :hint; \
-          expected :info, :info_long, :debug, false, or nil." % hint
-      end
-      
-      css_classes = TokenKinds
-      case options[:css]
-      when :class
-        @span_for_kind = Hash.new do |h, k|
-          if k.is_a? ::Symbol
-            kind = k_dup = k
-          else
-            kind = k.first
-            k_dup = k.dup
-          end
-          if kind != :space && (hint || css_class = css_classes[kind])
-            title = HTML.token_path_to_hint hint, k if hint
-            css_class ||= css_classes[kind]
-            h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
-          else
-            h[k_dup] = nil
-          end
-        end
-      when :style
-        @span_for_kind = Hash.new do |h, k|
-          kind = k.is_a?(Symbol) ? k : k.first
-          h[k.is_a?(Symbol) ? k : k.dup] =
-            if kind != :space && (hint || css_classes[kind])
-              title = HTML.token_path_to_hint hint, k if hint
-              style = @css.get_style Array(k).map { |c| css_classes[c] }
-              "<span#{title}#{" style=\"#{style}\"" if style}>"
-            end
-        end
-      else
-        raise ArgumentError, "Unknown value %p for :css." % options[:css]
-      end
+      @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
       
       @set_last_opened = options[:hint] || options[:css] == :style
     end
     
     def finish options
       unless @opened.empty?
-        warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
         @out << '</span>' while @opened.pop
         @last_opened = nil
       end
@@ -255,20 +216,10 @@ module Encoders
   public
     
     def text_token text, kind
-      if text =~ /#{HTML_ESCAPE_PATTERN}/o
-        text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
-      end
+      style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
       
-      style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
-      
-      if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
-        close = '</span>' * c
-        reopen = ''
-        @opened.each_with_index do |k, index|
-          reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
-        end
-        text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
-      end
+      text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
+      text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
       
       if style
         @out << style << text << '</span>'
@@ -279,25 +230,19 @@ module Encoders
     
     # token groups, eg. strings
     def begin_group kind
-      @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
       @opened << kind
       @last_opened = kind if @set_last_opened
     end
     
     def end_group kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a token (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
-      end
-      if @opened.pop
-        @out << '</span>'
-        @last_opened = @opened.last if @last_opened
-      end
+      check_group_nesting 'token group', kind if $CODERAY_DEBUG
+      close_span
     end
     
     # whole lines to be highlighted, eg. a deleted line in a diff
     def begin_line kind
-      if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
+      if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
         if style['class="']
           @out << style.sub('class="', 'class="line ')
         else
@@ -311,16 +256,71 @@ module Encoders
     end
     
     def end_line kind
-      if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
-        warn 'Malformed token stream: Trying to close a line (%p) ' \
-          'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
+      check_group_nesting 'line', kind if $CODERAY_DEBUG
+      close_span
+    end
+    
+  protected
+    
+    def check_options! options
+      unless [false, nil, :debug, :info, :info_long].include? options[:hint]
+        raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
+      end
+      
+      unless [:class, :style].include? options[:css]
+        raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
+      end
+      
+      options[:break_lines] = true if options[:line_numbers] == :inline
+    end
+    
+    def css_class_for_kinds kinds
+      TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+    end
+    
+    def style_for_kinds kinds
+      css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
+      @css.get_style_for_css_classes css_classes
+    end
+    
+    def make_span_for_kinds method, hint
+      Hash.new do |h, kinds|
+        h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
+          css_class = css_class_for_kinds(kinds)
+          title     = HTML.token_path_to_hint hint, kinds if hint
+          
+          if css_class || title
+            if method == :style
+              style = style_for_kinds(kinds)
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            else
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+            end
+          end
+        end
+      end
+    end
+    
+    def check_group_nesting name, kind
+      if @opened.empty? || @opened.last != kind
+        warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
       end
+    end
+    
+    def break_lines text, style
+      reopen = ''
+      @opened.each_with_index do |k, index|
+        reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
+      end
+      text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
+    end
+    
+    def close_span
       if @opened.pop
         @out << '</span>'
         @last_opened = @opened.last if @last_opened
       end
     end
-    
   end
   
 end

data/lib/coderay/encoders/html/css.rb

@@ -11,7 +11,7 @@ module Encoders
       end
 
       def initialize style = :default
-        @classes = Hash.new
+        @styles = Hash.new
         style = CSS.load_stylesheet style
         @stylesheet = [
           style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ module Encoders
         parse style::TOKEN_COLORS
       end
 
-      def get_style styles
-        cl = @classes[styles.first]
+      def get_style_for_css_classes css_classes
+        cl = @styles[css_classes.first]
         return '' unless cl
         style = ''
-        1.upto styles.size do |offset|
-          break if style = cl[styles[offset .. -1]]
+        1.upto css_classes.size do |offset|
+          break if style = cl[css_classes[offset .. -1]]
         end
         # warn 'Style not found: %p' % [styles] if style.empty?
         return style
@@ -52,8 +52,8 @@ module Encoders
           for selector in selectors.split(',')
             classes = selector.scan(/[-\w]+/)
             cl = classes.pop
-            @classes[cl] ||= Hash.new
-            @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
+            @styles[cl] ||= Hash.new
+            @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
           end
         end
       end