coderay 0.9.8 → 1.0.0

data/lib/coderay/encoders/text.rb

@@ -1,32 +1,46 @@
 module CodeRay
 module Encoders
-
+  
+  # Concats the tokens into a single string, resulting in the original
+  # code string if no tokens were removed.
+  # 
+  # Alias: +plain+, +plaintext+
+  # 
+  # == Options
+  # 
+  # === :separator
+  # A separator string to join the tokens.
+  # 
+  # Default: empty String
   class Text < Encoder
-
-    include Streamable
+    
     register_for :text
-
+    
     FILE_EXTENSION = 'txt'
-
+    
     DEFAULT_OPTIONS = {
-      :separator => ''
+      :separator => nil
     }
-
+    
+    def text_token text, kind
+      super
+      
+      if @first
+        @first = false
+      else
+        @out << @sep
+      end if @sep
+    end
+    
   protected
     def setup options
       super
+      
+      @first = true
       @sep = options[:separator]
     end
-
-    def text_token text, kind
-      text + @sep
-    end
-
-    def finish options
-      super.chomp @sep
-    end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/token_kind_filter.rb

@@ -0,0 +1,111 @@
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  # A Filter that selects tokens based on their token kind.
+  # 
+  # == Options
+  # 
+  # === :exclude
+  # 
+  # One or many symbols (in an Array) which shall be excluded.
+  # 
+  # Default: []
+  # 
+  # === :include
+  # 
+  # One or many symbols (in an array) which shall be included.
+  # 
+  # Default: :all, which means all tokens are included.
+  # 
+  # Exclusion wins over inclusion.
+  # 
+  # See also: CommentFilter
+  class TokenKindFilter < Filter
+    
+    register_for :token_kind_filter
+    
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+    
+  protected
+    def setup options
+      super
+      
+      @group_excluded = false
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+      include_group? kind
+    end
+    
+    def include_group? kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  public
+    
+    # Add the token to the output stream if +kind+ matches the conditions.
+    def text_token text, kind
+      super if !@group_excluded && include_text_token?(text, kind)
+    end
+    
+    # Add the token group to the output stream if +kind+ matches the
+    # conditions.
+    # 
+    # If it does not, all tokens inside the group are excluded from the
+    # stream, even if their kinds match.
+    def begin_group kind
+      if @group_excluded
+        @group_excluded += 1
+      elsif include_group? kind
+        super
+      else
+        @group_excluded = 1
+      end
+    end
+    
+    # See +begin_group+.
+    def begin_line kind
+      if @group_excluded
+        @group_excluded += 1
+      elsif include_group? kind
+        super
+      else
+        @group_excluded = 1
+      end
+    end
+    
+    # Take care of re-enabling the delegation of tokens to the output stream
+    # if an exluded group has ended.
+    def end_group kind
+      if @group_excluded
+        @group_excluded -= 1
+        @group_excluded = false if @group_excluded.zero?
+      else
+        super
+      end
+    end
+    
+    # See +end_group+.
+    def end_line kind
+      if @group_excluded
+        @group_excluded -= 1
+        @group_excluded = false if @group_excluded.zero?
+      else
+        super
+      end
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/encoders/xml.rb

@@ -1,39 +1,40 @@
 module CodeRay
 module Encoders
-
+  
   # = XML Encoder
   #
   # Uses REXML. Very slow.
   class XML < Encoder
-
-    include Streamable
+    
     register_for :xml
-
+    
     FILE_EXTENSION = 'xml'
-
-    require 'rexml/document'
-
+    
+    autoload :REXML, 'rexml/document'
+    
     DEFAULT_OPTIONS = {
       :tab_width => 8,
       :pretty => -1,
       :transitive => false,
     }
-
+    
   protected
-
     def setup options
+      super
+      
       @doc = REXML::Document.new
       @doc << REXML::XMLDecl.new
       @tab_width = options[:tab_width]
       @root = @node = @doc.add_element('coderay-tokens')
     end
-
+    
     def finish options
-      @out = ''
       @doc.write @out, options[:pretty], options[:transitive], true
-      @out
+      
+      super
     end
     
+  public
     def text_token text, kind
       if kind == :space
         token = @node
@@ -53,19 +54,19 @@ module Encoders
         end
       end
     end
-
-    def open_token kind
+    
+    def begin_group kind
       @node = @node.add_element kind.to_s
     end
-
-    def close_token kind
+    
+    def end_group kind
       if @node == @root
         raise 'no token to close!'
       end
       @node = @node.parent
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/encoders/yaml.rb

@@ -1,22 +1,50 @@
+autoload :YAML, 'yaml'
+
 module CodeRay
 module Encoders
-
+  
   # = YAML Encoder
   #
   # Slow.
   class YAML < Encoder
-
+    
     register_for :yaml
-
+    
     FILE_EXTENSION = 'yaml'
-
+    
   protected
-    def compile tokens, options
-      require 'yaml'
-      @out = tokens.to_a.to_yaml
+    def setup options
+      super
+      
+      @data = []
     end
-
+    
+    def finish options
+      output ::YAML.dump(@data)
+    end
+    
+  public
+    def text_token text, kind
+      @data << [text, kind]
+    end
+    
+    def begin_group kind
+      @data << [:begin_group, kind]
+    end
+    
+    def end_group kind
+      @data << [:end_group, kind]
+    end
+    
+    def begin_line kind
+      @data << [:begin_line, kind]
+    end
+    
+    def end_line kind
+      @data << [:end_line, kind]
+    end
+    
   end
-
+  
 end
 end

data/lib/coderay/for_redcloth.rb

@@ -30,7 +30,7 @@ module CodeRay
       end
       RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
       RedCloth::Formatters::HTML.module_eval do
-        def unescape(html)
+        def unescape(html)  # :nodoc:
           replacements = {
             '&' => '&',
             '"' => '"',
@@ -45,7 +45,7 @@ module CodeRay
           if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
             # simulating pre-4.2 behavior
             if opts[:text].sub!(/\A\[(\w+)\]/, '')
-              if CodeRay::Scanners[$1].plugin_id == 'plaintext'
+              if CodeRay::Scanners[$1].lang == :text
                 opts[:text] = $& + opts[:text]
               else
                 opts[:lang] = $1
@@ -57,7 +57,7 @@ module CodeRay
             @in_bc ||= nil
             format = @in_bc ? :div : :span
             opts[:text] = unescape(opts[:text]) unless @in_bc
-            highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
+            highlighted_code = CodeRay.encode opts[:text], opts[:lang], format
             highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
             highlighted_code
           else
@@ -74,7 +74,7 @@ module CodeRay
           @in_bc = nil
           opts[:lang] ? '' : "</pre>\n"
         end
-        def escape_pre(text)
+        def escape_pre(text)  # :nodoc:
           if @in_bc ||= nil
             text
           else

data/lib/coderay/helpers/file_type.rb

@@ -1,56 +1,70 @@
-#!/usr/bin/env ruby
 module CodeRay
-
-# = FileType
-#
-# A simple filetype recognizer.
-#
-# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
-#
-# License:: LGPL / ask the author
-# Version:: 0.1 (2005-09-01)
-#
-# == Documentation
-#
-#  # determine the type of the given
-#   lang = FileType[ARGV.first]
-#  
-#   # return :plaintext if the file type is unknown
-#   lang = FileType.fetch ARGV.first, :plaintext
-#  
-#   # try the shebang line, too
-#   lang = FileType.fetch ARGV.first, :plaintext, true
-module FileType
-
-  UnknownFileType = Class.new Exception
-
-  class << self
-
-    # Try to determine the file type of the file.
-    #
-    # +filename+ is a relative or absolute path to a file.
-    #
-    # The file itself is only accessed when +read_shebang+ is set to true.
-    # That means you can get filetypes from files that don't exist.
-    def [] filename, read_shebang = false
-      name = File.basename filename
-      ext = File.extname(name).sub(/^\./, '')  # from last dot, delete the leading dot
-      ext2 = filename.to_s[/\.(.*)/, 1]  # from first dot
-
-      type =
-        TypeFromExt[ext] ||
-        TypeFromExt[ext.downcase] ||
-        (TypeFromExt[ext2] if ext2) ||
-        (TypeFromExt[ext2.downcase] if ext2) ||
-        TypeFromName[name] ||
-        TypeFromName[name.downcase]
-      type ||= shebang(filename) if read_shebang
-
-      type
-    end
-
-    def shebang filename
-      begin
+  
+  # = FileType
+  #
+  # A simple filetype recognizer.
+  #
+  # == Usage
+  #
+  #  # determine the type of the given
+  #  lang = FileType[file_name]
+  #  
+  #  # return :text if the file type is unknown
+  #  lang = FileType.fetch file_name, :text
+  #  
+  #  # try the shebang line, too
+  #  lang = FileType.fetch file_name, :text, true
+  module FileType
+    
+    UnknownFileType = Class.new Exception
+    
+    class << self
+      
+      # Try to determine the file type of the file.
+      #
+      # +filename+ is a relative or absolute path to a file.
+      #
+      # The file itself is only accessed when +read_shebang+ is set to true.
+      # That means you can get filetypes from files that don't exist.
+      def [] filename, read_shebang = false
+        name = File.basename filename
+        ext = File.extname(name).sub(/^\./, '')  # from last dot, delete the leading dot
+        ext2 = filename.to_s[/\.(.*)/, 1]  # from first dot
+        
+        type =
+          TypeFromExt[ext] ||
+          TypeFromExt[ext.downcase] ||
+          (TypeFromExt[ext2] if ext2) ||
+          (TypeFromExt[ext2.downcase] if ext2) ||
+          TypeFromName[name] ||
+          TypeFromName[name.downcase]
+        type ||= shebang(filename) if read_shebang
+        
+        type
+      end
+      
+      # This works like Hash#fetch.
+      #
+      # If the filetype cannot be found, the +default+ value
+      # is returned.
+      def fetch filename, default = nil, read_shebang = false
+        if default && block_given?
+          warn 'Block supersedes default value argument; use either.'
+        end
+        
+        if type = self[filename, read_shebang]
+          type
+        else
+          return yield if block_given?
+          return default if default
+          raise UnknownFileType, 'Could not determine type of %p.' % filename
+        end
+      end
+      
+    protected
+      
+      def shebang filename
+        return unless File.exist? filename
         File.open filename, 'r' do |f|
           if first_line = f.gets
             if type = first_line[TypeFromShebang]
@@ -58,203 +72,70 @@ module FileType
             end
           end
         end
-      rescue IOError
-        nil
       end
+      
     end
-
-    # This works like Hash#fetch.
-    #
-    # If the filetype cannot be found, the +default+ value
-    # is returned.
-    def fetch filename, default = nil, read_shebang = false
-      if default and block_given?
-        warn 'block supersedes default value argument'
-      end
-
-      unless type = self[filename, read_shebang]
-        return yield if block_given?
-        return default if default
-        raise UnknownFileType, 'Could not determine type of %p.' % filename
-      end
-      type
+    
+    TypeFromExt = {
+      'c'        => :c,
+      'cfc'      => :xml,
+      'cfm'      => :xml,
+      'clj'      => :clojure,
+      'css'      => :css,
+      'diff'     => :diff,
+      'dpr'      => :delphi,
+      'gemspec'  => :ruby,
+      'groovy'   => :groovy,
+      'gvy'      => :groovy,
+      'h'        => :c,
+      'haml'     => :haml,
+      'htm'      => :page,
+      'html'     => :page,
+      'html.erb' => :erb,
+      'java'     => :java,
+      'js'       => :java_script,
+      'json'     => :json,
+      'mab'      => :ruby,
+      'pas'      => :delphi,
+      'patch'    => :diff,
+      'php'      => :php,
+      'php3'     => :php,
+      'php4'     => :php,
+      'php5'     => :php,
+      'prawn'    => :ruby,
+      'py'       => :python,
+      'py3'      => :python,
+      'pyw'      => :python,
+      'rake'     => :ruby,
+      'raydebug' => :raydebug,
+      'rb'       => :ruby,
+      'rbw'      => :ruby,
+      'rhtml'    => :erb,
+      'rjs'      => :ruby,
+      'rpdf'     => :ruby,
+      'ru'       => :ruby,
+      'rxml'     => :ruby,
+      # 'sch'      => :scheme,
+      'sql'      => :sql,
+      # 'ss'       => :scheme,
+      'xhtml'    => :page,
+      'xml'      => :xml,
+      'yaml'     => :yaml,
+      'yml'      => :yaml,
+    }
+    for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
+      TypeFromExt[cpp_alias] = :cpp
     end
-
+    
+    TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+    
+    TypeFromName = {
+      'Capfile'  => :ruby,
+      'Rakefile' => :ruby,
+      'Rantfile' => :ruby,
+      'Gemfile'  => :ruby,
+    }
+    
   end
-
-  TypeFromExt = {
-    'c'        => :c,
-    'cfc'      => :xml,
-    'cfm'      => :xml,
-    'css'      => :css,
-    'diff'     => :diff,
-    'dpr'      => :delphi,
-    'gemspec'  => :ruby,
-    'groovy'   => :groovy,
-    'gvy'      => :groovy,
-    'h'        => :c,
-    'htm'      => :html,
-    'html'     => :html,
-    'html.erb' => :rhtml,
-    'java'     => :java,
-    'js'       => :java_script,
-    'json'     => :json,
-    'mab'      => :ruby,
-    'pas'      => :delphi,
-    'patch'    => :diff,
-    'php'      => :php,
-    'php3'     => :php,
-    'php4'     => :php,
-    'php5'     => :php,
-    'py'       => :python,
-    'py3'      => :python,
-    'pyw'      => :python,
-    'rake'     => :ruby,
-    'raydebug' => :debug,
-    'rb'       => :ruby,
-    'rbw'      => :ruby,
-    'rhtml'    => :rhtml,
-    'rjs'      => :ruby,
-    'rpdf'     => :ruby,
-    'rxml'     => :ruby,
-    'sch'      => :scheme,
-    'sql'      => :sql,
-    'ss'       => :scheme,
-    'xhtml'    => :xhtml,
-    'xml'      => :xml,
-    'yaml'     => :yaml,
-    'yml'      => :yaml,
-  }
-  for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
-    TypeFromExt[cpp_alias] = :cpp
-  end
-
-  TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
-
-  TypeFromName = {
-    'Rakefile' => :ruby,
-    'Rantfile' => :ruby,
-  }
-
-end
-
-end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class FileTypeTests < Test::Unit::TestCase
   
-  include CodeRay
-  
-  def test_fetch
-    assert_raise FileType::UnknownFileType do
-      FileType.fetch ''
-    end
-
-    assert_throws :not_found do
-      FileType.fetch '.' do
-        throw :not_found
-      end
-    end
-
-    assert_equal :default, FileType.fetch('c', :default)
-
-    stderr, fake_stderr = $stderr, Object.new
-    $err = ''
-    def fake_stderr.write x
-      $err << x
-    end
-    $stderr = fake_stderr
-    FileType.fetch('c', :default) { }
-    assert_equal "block supersedes default value argument\n", $err
-    $stderr = stderr
-  end
-
-  def test_ruby
-    assert_equal :ruby, FileType['test.rb']
-    assert_equal :ruby, FileType['test.java.rb']
-    assert_equal :java, FileType['test.rb.java']
-    assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
-    assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
-    assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
-    assert_equal :ruby, FileType['./lib/tasks\repository.rake']
-    assert_not_equal :ruby, FileType['test_rb']
-    assert_not_equal :ruby, FileType['Makefile']
-    assert_not_equal :ruby, FileType['set.rb/set']
-    assert_not_equal :ruby, FileType['~/projects/blabla/rb']
-  end
-
-  def test_c
-    assert_equal :c, FileType['test.c']
-    assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
-    assert_not_equal :c, FileType['test_c']
-    assert_not_equal :c, FileType['Makefile']
-    assert_not_equal :c, FileType['set.h/set']
-    assert_not_equal :c, FileType['~/projects/blabla/c']
-  end
-
-  def test_cpp
-    assert_equal :cpp, FileType['test.c++']
-    assert_equal :cpp, FileType['test.cxx']
-    assert_equal :cpp, FileType['test.hh']
-    assert_equal :cpp, FileType['test.hpp']
-    assert_equal :cpp, FileType['test.cu']
-    assert_equal :cpp, FileType['test.C']
-    assert_not_equal :cpp, FileType['test.c']
-    assert_not_equal :cpp, FileType['test.h']
-  end
-
-  def test_html
-    assert_equal :html, FileType['test.htm']
-    assert_equal :xhtml, FileType['test.xhtml']
-    assert_equal :xhtml, FileType['test.html.xhtml']
-    assert_equal :rhtml, FileType['_form.rhtml']
-    assert_equal :rhtml, FileType['_form.html.erb']
-  end
-
-  def test_yaml
-    assert_equal :yaml, FileType['test.yml']
-    assert_equal :yaml, FileType['test.yaml']
-    assert_equal :yaml, FileType['my.html.yaml']
-    assert_not_equal :yaml, FileType['YAML']
-  end
-
-  def test_pathname
-    require 'pathname'
-    pn = Pathname.new 'test.rb'
-    assert_equal :ruby, FileType[pn]
-    dir = Pathname.new '/etc/var/blubb'
-    assert_equal :ruby, FileType[dir + pn]
-    assert_equal :cpp, FileType[dir + 'test.cpp']
-  end
-
-  def test_no_shebang
-    dir = './test'
-    if File.directory? dir
-      Dir.chdir dir do
-        assert_equal :c, FileType['test.c']
-      end
-    end
-  end
-  
-  def test_shebang_empty_file
-    require 'tmpdir'
-    tmpfile = File.join(Dir.tmpdir, 'bla')
-    File.open(tmpfile, 'w') { }  # touch
-    assert_equal nil, FileType[tmpfile]
-  end
-  
-  def test_shebang
-    require 'tmpdir'
-    tmpfile = File.join(Dir.tmpdir, 'bla')
-    File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
-    assert_equal :ruby, FileType[tmpfile, true]
-  end
-
 end