coderay 1.0.0.598.pre → 1.0.0.738.pre

data/lib/coderay/tokens.rb

@@ -1,5 +1,8 @@
 module CodeRay
-
+  
+  # GZip library for writing and reading token dumps.
+  autoload :GZip, 'coderay/helpers/gzip'
+  
   # = Tokens  TODO: Rewrite!
   #
   # The Tokens class represents a list of tokens returnd from
@@ -62,21 +65,21 @@ module CodeRay
     # options are passed to the encoder.
     def encode encoder, options = {}
       unless encoder.is_a? Encoders::Encoder
-        unless encoder.is_a? Class
+        if encoder.respond_to? :to_sym
           encoder_class = Encoders[encoder]
         end
         encoder = encoder_class.new options
       end
       encoder.encode_tokens self, options
     end
-
+    
     # Turn into a string using Encoders::Text.
     #
     # +options+ are passed to the encoder if given.
-    def to_s options = {}
-      encode :text, options
+    def to_s
+      encode CodeRay::Encoders::Encoder.new
     end
-
+    
     # Redirects unknown methods to encoder calls.
     #
     # For example, if you call +tokens.html+, the HTML encoder
@@ -104,27 +107,27 @@ module CodeRay
     # joined in one comment token by the Scanner.
     def optimize
       raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
-      last_kind = last_text = nil
-      new = self.class.new
-      for text, kind in self
-        if text.is_a? String
-          if kind == last_kind
-            last_text << text
-          else
-            new << [last_text, last_kind] if last_kind
-            last_text = text
-            last_kind = kind
-          end
-        else
-          new << [last_text, last_kind] if last_kind
-          last_kind = last_text = nil
-          new << [text, kind]
-        end
-      end
-      new << [last_text, last_kind] if last_kind
-      new
+      # last_kind = last_text = nil
+      # new = self.class.new
+      # for text, kind in self
+      #   if text.is_a? String
+      #     if kind == last_kind
+      #       last_text << text
+      #     else
+      #       new << [last_text, last_kind] if last_kind
+      #       last_text = text
+      #       last_kind = kind
+      #     end
+      #   else
+      #     new << [last_text, last_kind] if last_kind
+      #     last_kind = last_text = nil
+      #     new << [text, kind]
+      #   end
+      # end
+      # new << [last_text, last_kind] if last_kind
+      # new
     end
-
+    
     # Compact the object itself; see optimize.
     def optimize!
       replace optimize
@@ -135,30 +138,30 @@ module CodeRay
     # TODO: Test this!
     def fix
       raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
-      tokens = self.class.new
-      # Check token nesting using a stack of kinds.
-      opened = []
-      for type, kind in self
-        case type
-        when :begin_group
-          opened.push [:begin_group, kind]
-        when :begin_line
-          opened.push [:end_line, kind]
-        when :end_group, :end_line
-          expected = opened.pop
-          if [type, kind] != expected
-            # Unexpected end; decide what to do based on the kind:
-            # - token was never opened: delete the end (just skip it)
-            next unless opened.rindex expected
-            # - token was opened earlier: also close tokens in between
-            tokens << token until (token = opened.pop) == expected
-          end
-        end
-        tokens << [type, kind]
-      end
-      # Close remaining opened tokens
-      tokens << token while token = opened.pop
-      tokens
+      # tokens = self.class.new
+      # # Check token nesting using a stack of kinds.
+      # opened = []
+      # for type, kind in self
+      #   case type
+      #   when :begin_group
+      #     opened.push [:begin_group, kind]
+      #   when :begin_line
+      #     opened.push [:end_line, kind]
+      #   when :end_group, :end_line
+      #     expected = opened.pop
+      #     if [type, kind] != expected
+      #       # Unexpected end; decide what to do based on the kind:
+      #       # - token was never opened: delete the end (just skip it)
+      #       next unless opened.rindex expected
+      #       # - token was opened earlier: also close tokens in between
+      #       tokens << token until (token = opened.pop) == expected
+      #     end
+      #   end
+      #   tokens << [type, kind]
+      # end
+      # # Close remaining opened tokens
+      # tokens << token while token = opened.pop
+      # tokens
     end
     
     def fix!
@@ -177,7 +180,7 @@ module CodeRay
     def split_into_lines
       raise NotImplementedError
     end
-
+    
     def split_into_lines!
       replace split_into_lines
     end
@@ -239,19 +242,19 @@ module CodeRay
           when :end_group, :end_line
             opened.pop
           else
-            raise 'Unknown token action: %p, kind = %p' % [content, item]
+            raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item]
           end
           part << content << item
           content = nil
         else
-          raise 'else case reached'
+          raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item]
         end
       end
       parts << part
       parts << Tokens.new while parts.size < sizes.size
       parts
     end
-
+    
     # Dumps the object into a String that can be saved
     # in files or databases.
     #
@@ -268,9 +271,8 @@ module CodeRay
     #
     # See GZip module.
     def dump gzip_level = 7
-      require 'coderay/helpers/gzip_simple'
       dump = Marshal.dump self
-      dump = dump.gzip gzip_level
+      dump = GZip.gzip dump, gzip_level
       dump.extend Undumping
     end
     
@@ -278,7 +280,7 @@ module CodeRay
     def count
       size / 2
     end
-
+    
     # Include this module to give an object an #undump
     # method.
     #
@@ -289,70 +291,44 @@ module CodeRay
         Tokens.load self
       end
     end
-
+    
     # Undump the object using Marshal.load, then
     # unzip it using GZip.gunzip.
     #
     # The result is commonly a Tokens object, but
     # this is not guaranteed.
     def Tokens.load dump
-      require 'coderay/helpers/gzip_simple'
-      dump = dump.gunzip
+      dump = GZip.gunzip dump
       @dump = Marshal.load dump
     end
-
-    alias text_token push
-    def begin_group kind; push :begin_group, kind end
-    def end_group kind; push :end_group, kind end
-    def begin_line kind; push :begin_line, kind end
-    def end_line kind; push :end_line, kind end
-    alias tokens concat
     
-  end
-
-end
-
-if $0 == __FILE__
-  $VERBOSE = true
-  $: << File.join(File.dirname(__FILE__), '..')
-  eval DATA.read, nil, $0, __LINE__ + 4
-end
-
-__END__
-require 'test/unit'
-
-class TokensTest < Test::Unit::TestCase
-  
-  def test_creation
-    assert CodeRay::Tokens < Array
-    tokens = nil
-    assert_nothing_raised do
-      tokens = CodeRay::Tokens.new
-    end
-    assert_kind_of Array, tokens
-  end
-  
-  def test_adding_tokens
-    tokens = CodeRay::Tokens.new
-    assert_nothing_raised do
-      tokens.text_token 'string', :type
-      tokens.text_token '()', :operator
-    end
-    assert_equal tokens.size, 4
-    assert_equal tokens.count, 2
-  end
-  
-  def test_dump_undump
-    tokens = CodeRay::Tokens.new
-    assert_nothing_raised do
-      tokens.text_token 'string', :type
-      tokens.text_token '()', :operator
-    end
-    tokens2 = nil
-    assert_nothing_raised do
-      tokens2 = tokens.dump.undump
+    if defined?(RUBY_ENGINE) && RUBY_ENGINE['rbx']
+      #:nocov:
+      def text_token text, kind
+        self << text << kind
+      end
+      def begin_group kind
+        self << :begin_group << kind
+      end
+      def end_group kind
+        self << :end_group << kind
+      end
+      def begin_line kind
+        self << :begin_line << kind
+      end
+      def end_line kind
+        self << :end_line << kind
+      end
+      #:nocov:
+    else
+      alias text_token push
+      def begin_group kind; push :begin_group, kind end
+      def end_group kind; push :end_group, kind end
+      def begin_line kind; push :begin_line, kind end
+      def end_line kind; push :end_line, kind end
     end
-    assert_equal tokens, tokens2
+    alias tokens concat
+    
   end
   
-end
+end

data/test/functional/basic.rb

@@ -1,8 +1,21 @@
+# encoding: utf-8
 require 'test/unit'
 require 'coderay'
 
 class BasicTest < Test::Unit::TestCase
   
+  def assert_warning expected_warning
+    require 'stringio'
+    oldstderr = $stderr
+    $stderr = StringIO.new
+    yield
+    $stderr.rewind
+    given_warning = $stderr.read.chomp
+    assert_equal expected_warning, given_warning
+  ensure
+    $stderr = oldstderr
+  end
+  
   def test_version
     assert_nothing_raised do
       assert_match(/\A\d\.\d\.\d\z/, CodeRay::VERSION)
@@ -34,16 +47,40 @@ class BasicTest < Test::Unit::TestCase
     end
   end
   
+  def test_scan_file
+    CodeRay.scan_file __FILE__
+  end
+  
+  def test_encode
+    assert_equal 1, CodeRay.encode('test', :python, :count)
+  end
+  
+  def test_encode_tokens
+    assert_equal 1, CodeRay.encode_tokens(CodeRay::Tokens['test', :string], :count)
+  end
+  
+  def test_encode_file
+    assert_equal File.read(__FILE__), CodeRay.encode_file(__FILE__, :text)
+  end
+  
+  def test_highlight
+    assert_match '<div class="code"><pre>test</pre></div>', CodeRay.highlight('test', :python)
+  end
+  
+  def test_highlight_file
+    assert_match "require <span class=\"s\"><span class=\"dl\">'</span><span class=\"k\">test/unit</span><span class=\"dl\">'</span></span>\n", CodeRay.highlight_file(__FILE__)
+  end
+  
   def test_duo
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE))
+      CodeRay::Duo[:plain, :text].highlight(RUBY_TEST_CODE))
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain => :plain].highlight(RUBY_TEST_CODE))
+      CodeRay::Duo[:plain => :text].highlight(RUBY_TEST_CODE))
   end
   
   def test_duo_stream
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE, :stream => true))
+      CodeRay::Duo[:plain, :text].highlight(RUBY_TEST_CODE, :stream => true))
   end
   
   def test_comment_filter
@@ -107,12 +144,154 @@ more code  # and another comment, in-line.
   
   def test_list_of_encoders
     assert_kind_of(Array, CodeRay::Encoders.list)
-    assert CodeRay::Encoders.list.include?('count')
+    assert CodeRay::Encoders.list.include?(:count)
   end
   
   def test_list_of_scanners
     assert_kind_of(Array, CodeRay::Scanners.list)
-    assert CodeRay::Scanners.list.include?('plaintext')
+    assert CodeRay::Scanners.list.include?(:plaintext)
+  end
+  
+  def test_token_kinds
+    assert_kind_of Hash, CodeRay::TokenKinds
+    for kind, css_class in CodeRay::TokenKinds
+      assert_kind_of Symbol, kind
+      if css_class != false
+        assert_kind_of String, css_class, "TokenKinds[%p] == %p" % [kind, css_class]
+      end
+    end
+    assert_equal 'r', CodeRay::TokenKinds[:reserved]
+    assert_equal false, CodeRay::TokenKinds[:shibboleet]
+  end
+  
+  class Milk < CodeRay::Encoders::Encoder
+    FILE_EXTENSION = 'cocoa'
+  end
+  
+  class HoneyBee < CodeRay::Encoders::Encoder
+  end
+  
+  def test_encoder_file_extension
+    assert_nothing_raised do
+      assert_equal 'html', CodeRay::Encoders::HTML::FILE_EXTENSION
+      assert_equal 'cocoa', Milk::FILE_EXTENSION
+      assert_equal 'cocoa', Milk.new.file_extension
+      assert_equal 'honeybee', HoneyBee::FILE_EXTENSION
+      assert_equal 'honeybee', HoneyBee.new.file_extension
+    end
+    assert_raise NameError do
+      HoneyBee::MISSING_CONSTANT
+    end
+  end
+  
+  def test_encoder_tokens
+    encoder = CodeRay::Encoders::Encoder.new
+    encoder.send :setup, {}
+    assert_raise(ArgumentError) { encoder.token :strange, '' }
+    encoder.token 'test', :debug
+  end
+  
+  def test_encoder_deprecated_interface
+    encoder = CodeRay::Encoders::Encoder.new
+    encoder.send :setup, {}
+    assert_warning 'Using old Tokens#<< interface.' do
+      encoder << ['test', :content]
+    end
+    assert_raise ArgumentError do
+      encoder << [:strange, :input]
+    end
+    assert_raise ArgumentError do
+      encoder.encode_tokens [['test', :token]]
+    end
+  end
+  
+  def encoder_token_interface_deprecation_warning_given
+    CodeRay::Encoders::Encoder.send :class_variable_get, :@@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
+  end
+  
+  def test_scanner_file_extension
+    assert_equal 'rb', CodeRay::Scanners::Ruby.file_extension
+    assert_equal 'rb', CodeRay::Scanners::Ruby.new.file_extension
+    assert_equal 'java', CodeRay::Scanners::Java.file_extension
+    assert_equal 'java', CodeRay::Scanners::Java.new.file_extension
+  end
+  
+  def test_scanner_lang
+    assert_equal :ruby, CodeRay::Scanners::Ruby.lang
+    assert_equal :ruby, CodeRay::Scanners::Ruby.new.lang
+    assert_equal :java, CodeRay::Scanners::Java.lang
+    assert_equal :java, CodeRay::Scanners::Java.new.lang
+  end
+  
+  def test_scanner_tokenize
+    assert_equal ['foo', :plain], CodeRay::Scanners::Plain.new.tokenize('foo')
+    assert_equal [['foo', :plain], ['bar', :plain]], CodeRay::Scanners::Plain.new.tokenize(['foo', 'bar'])
+    assert_raise ArgumentError do
+      CodeRay::Scanners::Plain.new.tokenize 42
+    end
+  end
+  
+  def test_scanner_tokens
+    scanner = CodeRay::Scanners::Plain.new
+    scanner.tokenize('foo')
+    assert_equal ['foo', :plain], scanner.tokens
+    scanner.string = ''
+    assert_equal ['', :plain], scanner.tokens
+  end
+  
+  def test_scanner_line_and_column
+    scanner = CodeRay::Scanners::Plain.new "foo\nbär+quux"
+    assert_equal 0, scanner.pos
+    assert_equal 1, scanner.line
+    assert_equal 0, scanner.column
+    scanner.scan(/foo\nbär/)
+    assert_equal 8, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 5, scanner.column
+  end
+  
+  def test_scanner_use_subclasses
+    assert_raise NotImplementedError do
+      CodeRay::Scanners::Scanner.new
+    end
+  end
+  
+  class InvalidScanner < CodeRay::Scanners::Scanner
+  end
+  
+  def test_scanner_scan_tokens
+    assert_raise NotImplementedError do
+      InvalidScanner.new.tokenize ''
+    end
+  end
+  
+  class RaisingScanner < CodeRay::Scanners::Scanner
+    def scan_tokens encoder, options
+      raise_inspect 'message', [], :initial
+    end
+  end
+  
+  def test_scanner_raise_inspect
+    assert_raise CodeRay::Scanners::Scanner::ScanError do
+      RaisingScanner.new.tokenize ''
+    end
+  end
+  
+  def test_scan_a_frozen_string
+    assert_nothing_raised do
+      CodeRay.scan RUBY_VERSION, :ruby
+      CodeRay.scan RUBY_VERSION, :plain
+    end
+  end
+  
+  def test_scan_a_non_string
+    assert_nothing_raised do
+      CodeRay.scan 42, :ruby
+      CodeRay.scan nil, :ruby
+      CodeRay.scan self, :ruby
+      CodeRay.encode ENV.to_hash, :ruby, :page
+      CodeRay.highlight CodeRay, :plain
+    end
   end
   
 end