coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay.rb

@@ -1,21 +1,16 @@
-# encoding: utf-8
-# Encoding.default_internal = 'UTF-8'
-
 # = CodeRay Library
 #
 # CodeRay is a Ruby library for syntax highlighting.
 #
-# I try to make CodeRay easy to use and intuitive, but at the same time fully
-# featured, complete, fast and efficient.
+# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
+# fast and efficient.
 # 
 # See README.
 # 
 # It consists mainly of
-# * the main engine: CodeRay (Scanners::Scanner, Tokens, Encoders::Encoder)
-# * the plugin system: PluginHost, Plugin
+# * the main engine: CodeRay (Scanners::Scanner, Tokens, Encoders::Encoder), PluginHost
 # * the scanners in CodeRay::Scanners
 # * the encoders in CodeRay::Encoders
-# * the styles in CodeRay::Styles
 # 
 # Here's a fancy graphic to light up this gray docu:
 # 
@@ -27,8 +22,8 @@
 #
 # == Usage
 #
-# Remember you need RubyGems to use CodeRay, unless you have it in your load
-# path. Run Ruby with -rubygems option if required.
+# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
+# -rubygems option if required.
 #
 # === Highlight Ruby code in a string as html
 # 
@@ -127,30 +122,22 @@ module CodeRay
   
   $CODERAY_DEBUG ||= false
   
-  require 'coderay/version'
-  
-  # helpers
-  autoload :FileType, 'coderay/helpers/file_type'
-  
-  # Tokens
-  autoload :Tokens, 'coderay/tokens'
-  autoload :TokensProxy, 'coderay/tokens_proxy'
-  autoload :TokenKinds, 'coderay/token_kinds'
-  
-  # Plugin system
-  autoload :PluginHost, 'coderay/helpers/plugin'
-  autoload :Plugin, 'coderay/helpers/plugin'
-  
-  # Plugins
-  autoload :Scanners, 'coderay/scanner'
-  autoload :Encoders, 'coderay/encoder'
-  autoload :Styles, 'coderay/style'
-  
-  # Convenience access and reusable Encoder/Scanner pair
-  autoload :Duo, 'coderay/duo'
-  
+  # Version: Major.Minor.Teeny[.Revision]
+  # Major: 0 for pre-stable, 1 for stable
+  # Minor: feature milestone
+  # Teeny: development state, 0 for pre-release
+  # Revision: Subversion Revision number (generated on rake gem:make)
+  VERSION = '1.0.0'
+
+  require 'coderay/tokens'
+  require 'coderay/token_kinds'
+  require 'coderay/scanner'
+  require 'coderay/encoder'
+  require 'coderay/duo'
+  require 'coderay/style'
+
   class << self
-    
+
     # Scans the given +code+ (a String) with the Scanner for +lang+.
     #
     # This is a simple way to use CodeRay. Example:
@@ -159,15 +146,15 @@ module CodeRay
     #
     # See also demo/demo_simple.
     def scan code, lang, options = {}, &block
-      # FIXME: return a proxy for direct-stream encoding
-      TokensProxy.new code, lang, options, block
+      scanner = Scanners[lang].new code, options, &block
+      scanner.tokenize
     end
-    
+
     # Scans +filename+ (a path to a code file) with the Scanner for +lang+.
     #
     # If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
     # determine it. If it cannot find out what type it is, it uses
-    # CodeRay::Scanners::Text.
+    # CodeRay::Scanners::Plaintext.
     #
     # Calls CodeRay.scan.
     #
@@ -175,22 +162,35 @@ module CodeRay
     #  require 'coderay'
     #  page = CodeRay.scan_file('some_c_code.c').html
     def scan_file filename, lang = :auto, options = {}, &block
-      lang = FileType.fetch filename, :text, true if lang == :auto
-      code = File.read filename
-      scan code, lang, options, &block
+      file = IO.read filename
+      if lang == :auto
+        require 'coderay/helpers/file_type'
+        lang = FileType.fetch filename, :plaintext, true
+      end
+      scan file, lang, options = {}, &block
     end
-    
+
     # Encode a string.
     #
     # This scans +code+ with the the Scanner for +lang+ and then
     # encodes it with the Encoder for +format+.
     # +options+ will be passed to the Encoder.
     #
-    # See CodeRay::Encoder.encode.
+    # See CodeRay::Encoder.encode
     def encode code, lang, format, options = {}
       encoder(format, options).encode code, lang, options
     end
-    
+
+    # Highlight a string into a HTML <div>.
+    #
+    # CSS styles use classes, so you have to include a stylesheet
+    # in your output.
+    #
+    # See encode.
+    def highlight code, lang, options = { :css => :class }, format = :div
+      encode code, lang, format, options
+    end
+
     # Encode pre-scanned Tokens.
     # Use this together with CodeRay.scan:
     #
@@ -203,7 +203,7 @@ module CodeRay
     def encode_tokens tokens, format, options = {}
       encoder(format, options).encode_tokens tokens, options
     end
-    
+
     # Encodes +filename+ (a path to a code file) with the Scanner for +lang+.
     #
     # See CodeRay.scan_file.
@@ -216,17 +216,7 @@ module CodeRay
       tokens = scan_file filename, :auto, get_scanner_options(options)
       encode_tokens tokens, format, options
     end
-    
-    # Highlight a string into a HTML <div>.
-    #
-    # CSS styles use classes, so you have to include a stylesheet
-    # in your output.
-    #
-    # See encode.
-    def highlight code, lang, options = { :css => :class }, format = :div
-      encode code, lang, format, options
-    end
-    
+
     # Highlight a file into a HTML <div>.
     #
     # CSS styles use classes, so you have to include a stylesheet
@@ -236,7 +226,7 @@ module CodeRay
     def highlight_file filename, options = { :css => :class }, format = :div
       encode_file filename, format, options
     end
-    
+
     # Finds the Encoder class for +format+ and creates an instance, passing
     # +options+ to it.
     #
@@ -254,15 +244,15 @@ module CodeRay
     def encoder format, options = {}
       Encoders[format].new options
     end
-    
+
     # Finds the Scanner class for +lang+ and creates an instance, passing
     # +options+ to it.
     #
     # See Scanner.new.
-    def scanner lang, options = {}, &block
-      Scanners[lang].new '', options, &block
+    def scanner lang, options = {}
+      Scanners[lang].new '', options
     end
-    
+
     # Extract the options for the scanner from the +options+ hash.
     #
     # Returns an empty Hash if <tt>:scanner_options</tt> is not set.
@@ -272,7 +262,15 @@ module CodeRay
     def get_scanner_options options
       options.fetch :scanner_options, {}
     end
-    
+
   end
-  
+
+end
+
+# Run a test script.
+if $0 == __FILE__
+  $stderr.print 'Press key to print demo.'; gets
+  # Just use this file as an example of Ruby code.
+  code = File.read(__FILE__)[/module CodeRay.*/m]
+  print CodeRay.scan(code, :ruby).html
 end

data/lib/coderay/duo.rb

@@ -1,3 +1,4 @@
+($:.unshift '..'; require 'coderay') unless defined? CodeRay
 module CodeRay
   
   # = Duo
@@ -21,7 +22,7 @@ module CodeRay
     # Create a new Duo, holding a lang and a format to highlight code.
     # 
     # simple:
-    #   CodeRay::Duo[:ruby, :html].highlight 'bla 42'
+    #   CodeRay::Duo[:ruby, :page].highlight 'bla 42'
     # 
     # with options:
     #   CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
@@ -35,7 +36,7 @@ module CodeRay
     # The options are forwarded to scanner and encoder
     # (see CodeRay.get_scanner_options).
     def initialize lang = nil, format = nil, options = {}
-      if format.nil? && lang.is_a?(Hash) && lang.size == 1
+      if format == nil and lang.is_a? Hash and lang.size == 1
         @lang = lang.keys.first
         @format = lang[@lang]
       else
@@ -79,3 +80,55 @@ module CodeRay
   end
   
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class DuoTest < Test::Unit::TestCase
+  
+  def test_two_arguments
+    duo = CodeRay::Duo[:ruby, :html]
+    assert_kind_of CodeRay::Scanners[:ruby], duo.scanner
+    assert_kind_of CodeRay::Encoders[:html], duo.encoder
+  end
+  
+  def test_two_hash
+    duo = CodeRay::Duo[:ruby => :html]
+    assert_kind_of CodeRay::Scanners[:ruby], duo.scanner
+    assert_kind_of CodeRay::Encoders[:html], duo.encoder
+  end
+  
+  def test_call
+    duo = CodeRay::Duo[:python => :yaml]
+    assert_equal <<-'YAML', duo.call('def test: "pass"')
+--- 
+- - def
+  - :keyword
+- - " "
+  - :space
+- - test
+  - :method
+- - ":"
+  - :operator
+- - " "
+  - :space
+- - :begin_group
+  - :string
+- - "\""
+  - :delimiter
+- - pass
+  - :content
+- - "\""
+  - :delimiter
+- - :end_group
+  - :string
+    YAML
+  end
+  
+end

data/lib/coderay/encoder.rb

@@ -1,5 +1,5 @@
 module CodeRay
-  
+
   # This module holds the Encoder class and its subclasses.
   # For example, the HTML encoder is named CodeRay::Encoders::HTML
   # can be found in coderay/encoders/html.
@@ -8,10 +8,9 @@ module CodeRay
   # mechanism and the [] method that returns the Encoder class
   # belonging to the given format.
   module Encoders
-    
     extend PluginHost
     plugin_path File.dirname(__FILE__), 'encoders'
-    
+
     # = Encoder
     #
     # The Encoder base class. Together with Scanner and
@@ -27,32 +26,27 @@ module CodeRay
     class Encoder
       extend Plugin
       plugin_host Encoders
-      
+
       class << self
-        
+
         # If FILE_EXTENSION isn't defined, this method returns the
         # downcase class name instead.
         def const_missing sym
           if sym == :FILE_EXTENSION
-            (defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
+            plugin_id
           else
             super
           end
         end
-        
-        # The default file extension for output file of this encoder class.
-        def file_extension
-          self::FILE_EXTENSION
-        end
-        
+
       end
-      
+
       # Subclasses are to store their default options in this constant.
       DEFAULT_OPTIONS = { }
-      
+
       # The options you gave the Encoder at creating.
-      attr_accessor :options, :scanner
-      
+      attr_accessor :options
+
       # Creates a new Encoder.
       # +options+ is saved and used for all encode operations, as long
       # as you don't overwrite it there by passing additional options.
@@ -65,43 +59,55 @@ module CodeRay
       # added to the options you passed at creation.
       def initialize options = {}
         @options = self.class::DEFAULT_OPTIONS.merge options
-        @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = false
+        raise "I am only the basic Encoder class. I can't encode "\
+          "anything. :( Use my subclasses." if self.class == Encoder
+        $ALREADY_WARNED_OLD_INTERFACE = false
       end
-      
+
       # Encode a Tokens object.
       def encode_tokens tokens, options = {}
         options = @options.merge options
-        @scanner = tokens.scanner if tokens.respond_to? :scanner
         setup options
         compile tokens, options
         finish options
       end
-      
+
       # Encode the given +code+ using the Scanner for +lang+.
       def encode code, lang, options = {}
         options = @options.merge options
-        @scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
         setup options
-        @scanner.tokenize
+        scanner_options = CodeRay.get_scanner_options options
+        scanner_options[:tokens] = self
+        CodeRay.scan code, lang, scanner_options
         finish options
       end
-      
+
       # You can use highlight instead of encode, if that seems
       # more clear to you.
       alias highlight encode
-      
-      # The default file extension for this encoder.
+
+      # Return the default file extension for outputs of this encoder.
       def file_extension
-        self.class.file_extension
+        self.class::FILE_EXTENSION
       end
       
       def << token
-        unless @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
-          warn 'Using old Tokens#<< interface.'
-          @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = true
-        end
+        warn 'Using old Tokens#<< interface.' unless $ALREADY_WARNED_OLD_INTERFACE
+        $ALREADY_WARNED_OLD_INTERFACE = true
         self.token(*token)
       end
+
+    protected
+
+      # Called with merged options before encoding starts.
+      # Sets @out to an empty string.
+      #
+      # See the HTML Encoder for an example of option caching.
+      def setup options
+        @out = ''
+      end
+      
+    public
       
       # Called with +content+ and +kind+ of the currently scanned token.
       # For simple scanners, it's enougth to implement this method.
@@ -121,13 +127,12 @@ module CodeRay
         when :end_line
           end_line kind
         else
-          raise ArgumentError, 'Unknown token content type: %p, kind = %p' % [content, kind]
+          raise 'Unknown token content type: %p, kind = %p' % [content, kind]
         end
       end
       
       # Called for each text token ([text, kind]), where text is a String.
       def text_token text, kind
-        @out << text
       end
       
       # Starts a token group with the given +kind+.
@@ -148,24 +153,6 @@ module CodeRay
       
     protected
       
-      # Called with merged options before encoding starts.
-      # Sets @out to an empty string.
-      #
-      # See the HTML Encoder for an example of option caching.
-      def setup options
-        @out = get_output(options)
-      end
-      
-      def get_output options
-        options[:out] || ''
-      end
-      
-      # Append data.to_s to the output. Returns the argument.
-      def output data
-        @out << data.to_s
-        data
-      end
-      
       # Called with merged options after encoding starts.
       # The return value is the result of encoding, typically @out.
       def finish options
@@ -191,11 +178,11 @@ module CodeRay
         end
         raise 'odd number list for Tokens' if content
       end
-      
+    
       alias tokens compile
       public :tokens
       
     end
-    
+
   end
 end