coderay 0.9.8 → 1.0.0

data/lib/coderay/duo.rb

@@ -21,10 +21,7 @@ module CodeRay
     # Create a new Duo, holding a lang and a format to highlight code.
     # 
     # simple:
-    #   CodeRay::Duo[:ruby, :page].highlight 'bla 42'
-    # 
-    # streaming:
-    #   CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
+    #   CodeRay::Duo[:ruby, :html].highlight 'bla 42'
     # 
     # with options:
     #   CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
@@ -38,7 +35,7 @@ module CodeRay
     # The options are forwarded to scanner and encoder
     # (see CodeRay.get_scanner_options).
     def initialize lang = nil, format = nil, options = {}
-      if format == nil and lang.is_a? Hash and lang.size == 1
+      if format.nil? && lang.is_a?(Hash) && lang.size == 1
         @lang = lang.keys.first
         @format = lang[@lang]
       else
@@ -47,12 +44,12 @@ module CodeRay
       end
       @options = options
     end
-
+    
     class << self
       # To allow calls like Duo[:ruby, :html].highlight.
       alias [] new
     end
-
+    
     # The scanner of the duo. Only created once.
     def scanner
       @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
@@ -64,22 +61,21 @@ module CodeRay
     end
     
     # Tokenize and highlight the code using +scanner+ and +encoder+.
-    #
-    # If the :stream option is set, the Duo will go into streaming mode,
-    # saving memory for the cost of time.
-    def encode code, options = { :stream => false }
-      stream = options.delete :stream
+    def encode code, options = {}
       options = @options.merge options
-      if stream
-        encoder.encode_stream(code, @lang, options)
-      else
-        scanner.code = code
-        encoder.encode_tokens(scanner.tokenize, options)
-      end
+      encoder.encode(code, @lang, options)
     end
     alias highlight encode
-
+    
+    # Allows to use Duo like a proc object:
+    # 
+    #  CodeRay::Duo[:python => :yaml].call(code)
+    # 
+    # or, in Ruby 1.9 and later:
+    # 
+    #  CodeRay::Duo[:python => :yaml].(code)
+    alias call encode
+    
   end
-
+  
 end
-

data/lib/coderay/encoder.rb

@@ -1,5 +1,5 @@
 module CodeRay
-
+  
   # This module holds the Encoder class and its subclasses.
   # For example, the HTML encoder is named CodeRay::Encoders::HTML
   # can be found in coderay/encoders/html.
@@ -8,9 +8,10 @@ module CodeRay
   # mechanism and the [] method that returns the Encoder class
   # belonging to the given format.
   module Encoders
+    
     extend PluginHost
     plugin_path File.dirname(__FILE__), 'encoders'
-
+    
     # = Encoder
     #
     # The Encoder base class. Together with Scanner and
@@ -26,34 +27,32 @@ module CodeRay
     class Encoder
       extend Plugin
       plugin_host Encoders
-
-      attr_reader :token_stream
-
+      
       class << self
-
-        # Returns if the Encoder can be used in streaming mode.
-        def streamable?
-          is_a? Streamable
-        end
-
+        
         # If FILE_EXTENSION isn't defined, this method returns the
         # downcase class name instead.
         def const_missing sym
           if sym == :FILE_EXTENSION
-            plugin_id
+            (defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
           else
             super
           end
         end
-
+        
+        # The default file extension for output file of this encoder class.
+        def file_extension
+          self::FILE_EXTENSION
+        end
+        
       end
-
+      
       # Subclasses are to store their default options in this constant.
-      DEFAULT_OPTIONS = { :stream => false }
-
+      DEFAULT_OPTIONS = { }
+      
       # The options you gave the Encoder at creating.
-      attr_accessor :options
-
+      attr_accessor :options, :scanner
+      
       # Creates a new Encoder.
       # +options+ is saved and used for all encode operations, as long
       # as you don't overwrite it there by passing additional options.
@@ -61,153 +60,142 @@ module CodeRay
       # Encoder objects provide three encode methods:
       # - encode simply takes a +code+ string and a +lang+
       # - encode_tokens expects a +tokens+ object instead
-      # - encode_stream is like encode, but uses streaming mode.
       #
       # Each method has an optional +options+ parameter. These are
       # added to the options you passed at creation.
       def initialize options = {}
         @options = self.class::DEFAULT_OPTIONS.merge options
-        raise "I am only the basic Encoder class. I can't encode "\
-          "anything. :( Use my subclasses." if self.class == Encoder
+        @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = false
       end
-
+      
       # Encode a Tokens object.
       def encode_tokens tokens, options = {}
         options = @options.merge options
+        @scanner = tokens.scanner if tokens.respond_to? :scanner
         setup options
         compile tokens, options
         finish options
       end
-
-      # Encode the given +code+ after tokenizing it using the Scanner
-      # for +lang+.
+      
+      # Encode the given +code+ using the Scanner for +lang+.
       def encode code, lang, options = {}
         options = @options.merge options
-        scanner_options = CodeRay.get_scanner_options(options)
-        tokens = CodeRay.scan code, lang, scanner_options
-        encode_tokens tokens, options
+        @scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
+        setup options
+        @scanner.tokenize
+        finish options
       end
-
+      
       # You can use highlight instead of encode, if that seems
       # more clear to you.
       alias highlight encode
-
-      # Encode the given +code+ using the Scanner for +lang+ in
-      # streaming mode.
-      def encode_stream code, lang, options = {}
-        raise NotStreamableError, self unless kind_of? Streamable
-        options = @options.merge options
-        setup options
-        scanner_options = CodeRay.get_scanner_options options
-        @token_stream =
-          CodeRay.scan_stream code, lang, scanner_options, &self
-        finish options
-      end
-
-      # Behave like a proc. The token method is converted to a proc.
-      def to_proc
-        method(:token).to_proc
-      end
-
-      # Return the default file extension for outputs of this encoder.
+      
+      # The default file extension for this encoder.
       def file_extension
-        self.class::FILE_EXTENSION
+        self.class.file_extension
       end
-
-    protected
-
-      # Called with merged options before encoding starts.
-      # Sets @out to an empty string.
-      #
-      # See the HTML Encoder for an example of option caching.
-      def setup options
-        @out = ''
+      
+      def << token
+        unless @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
+          warn 'Using old Tokens#<< interface.'
+          @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = true
+        end
+        self.token(*token)
       end
-
+      
       # Called with +content+ and +kind+ of the currently scanned token.
       # For simple scanners, it's enougth to implement this method.
       #
-      # By default, it calls text_token or block_token, depending on
-      # whether +content+ is a String.
+      # By default, it calls text_token, begin_group, end_group, begin_line,
+      # or end_line, depending on the +content+.
       def token content, kind
-        encoded_token =
-          if content.is_a? ::String
-            text_token content, kind
-          elsif content.is_a? ::Symbol
-            block_token content, kind
-          else
-            raise 'Unknown token content type: %p' % [content]
-          end
-        append_encoded_token_to_output encoded_token
-      end
-      
-      def append_encoded_token_to_output encoded_token
-        @out << encoded_token if encoded_token && defined?(@out) && @out
-      end
-      
-      # Called for each text token ([text, kind]), where text is a String.
-      def text_token text, kind
-      end
-      
-      # Called for each block (non-text) token ([action, kind]),
-      # where +action+ is a Symbol.
-      # 
-      # Calls open_token, close_token, begin_line, and end_line according to
-      # the value of +action+.
-      def block_token action, kind
-        case action
-        when :open
-          open_token kind
-        when :close
-          close_token kind
+        case content
+        when String
+          text_token content, kind
+        when :begin_group
+          begin_group kind
+        when :end_group
+          end_group kind
         when :begin_line
           begin_line kind
         when :end_line
           end_line kind
         else
-          raise 'unknown block action: %p' % action
+          raise ArgumentError, 'Unknown token content type: %p, kind = %p' % [content, kind]
         end
       end
       
-      # Called for each block token at the start of the block ([:open, kind]).
-      def open_token kind
+      # Called for each text token ([text, kind]), where text is a String.
+      def text_token text, kind
+        @out << text
       end
       
-      # Called for each block token end of the block ([:close, kind]).
-      def close_token kind
+      # Starts a token group with the given +kind+.
+      def begin_group kind
       end
       
-      # Called for each line token block at the start of the line ([:begin_line, kind]).
+      # Ends a token group with the given +kind+.
+      def end_group kind
+      end
+      
+      # Starts a new line token group with the given +kind+.
       def begin_line kind
       end
       
-      # Called for each line token block at the end of the line ([:end_line, kind]).
+      # Ends a new line token group with the given +kind+.
       def end_line kind
       end
-
+      
+    protected
+      
+      # Called with merged options before encoding starts.
+      # Sets @out to an empty string.
+      #
+      # See the HTML Encoder for an example of option caching.
+      def setup options
+        @out = get_output(options)
+      end
+      
+      def get_output options
+        options[:out] || ''
+      end
+      
+      # Append data.to_s to the output. Returns the argument.
+      def output data
+        @out << data.to_s
+        data
+      end
+      
       # Called with merged options after encoding starts.
       # The return value is the result of encoding, typically @out.
       def finish options
         @out
       end
-
+      
       # Do the encoding.
       #
-      # The already created +tokens+ object must be used; it can be a
-      # TokenStream or a Tokens object.
-      if RUBY_VERSION >= '1.9'
-        def compile tokens, options
-          for text, kind in tokens
-            token text, kind
+      # The already created +tokens+ object must be used; it must be a
+      # Tokens object.
+      def compile tokens, options = {}
+        content = nil
+        for item in tokens
+          if item.is_a? Array
+            raise ArgumentError, 'Two-element array tokens are no longer supported.'
+          end
+          if content
+            token content, item
+            content = nil
+          else
+            content = item
           end
         end
-      else
-        def compile tokens, options
-          tokens.each(&self)
-        end
+        raise 'odd number list for Tokens' if content
       end
-
+      
+      alias tokens compile
+      public :tokens
+      
     end
-
+    
   end
 end

data/lib/coderay/encoders/_map.rb

@@ -1,12 +1,17 @@
 module CodeRay
 module Encoders
-
+  
   map \
-    :loc => :lines_of_code,
-    :plain => :text,
-    :stats => :statistic,
-    :terminal => :term,
-    :tex => :latex
-
+    :loc             => :lines_of_code,
+    :plain           => :text,
+    :plaintext       => :text,
+    :remove_comments => :comment_filter,
+    :stats           => :statistic,
+    :term            => :terminal,
+    :tty             => :terminal,
+    :yml             => :yaml
+  
+  # No default because Tokens#nonsense should raise NoMethodError.
+  
 end
 end

data/lib/coderay/encoders/comment_filter.rb

@@ -1,43 +1,25 @@
-($:.unshift '../..'; require 'coderay') unless defined? CodeRay
 module CodeRay
 module Encoders
   
-  load :token_class_filter
+  load :token_kind_filter
   
-  class CommentFilter < TokenClassFilter
+  # A simple Filter that removes all tokens of the :comment kind.
+  # 
+  # Alias: +remove_comments+
+  # 
+  # Usage:
+  #  CodeRay.scan('print # foo', :ruby).comment_filter.text
+  #  #-> "print "
+  # 
+  # See also: TokenKindFilter, LinesOfCode
+  class CommentFilter < TokenKindFilter
     
     register_for :comment_filter
     
     DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
-      :exclude => [:comment]
+      :exclude => [:comment, :docstring]
     
   end
   
 end
 end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  $: << File.join(File.dirname(__FILE__), '..')
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class CommentFilterTest < Test::Unit::TestCase
-  
-  def test_filtering_comments
-    tokens = CodeRay.scan <<-RUBY, :ruby
-#!/usr/bin/env ruby
-# a minimal Ruby program
-puts "Hello world!"
-    RUBY
-    assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
-#!/usr/bin/env ruby
-
-puts "Hello world!"
-    RUBY_FILTERED
-  end
-  
-end