coderay 0.9.8 → 1.0.0

data/lib/coderay/tokens_proxy.rb

@@ -0,0 +1,55 @@
+module CodeRay
+  
+  # The result of a scan operation is a TokensProxy, but should act like Tokens.
+  # 
+  # This proxy makes it possible to use the classic CodeRay.scan.encode API
+  # while still providing the benefits of direct streaming.
+  class TokensProxy
+    
+    attr_accessor :input, :lang, :options, :block
+    
+    # Create a new TokensProxy with the arguments of CodeRay.scan.
+    def initialize input, lang, options = {}, block = nil
+      @input   = input
+      @lang    = lang
+      @options = options
+      @block   = block
+    end
+    
+    # Call CodeRay.encode if +encoder+ is a Symbol;
+    # otherwise, convert the receiver to tokens and call encoder.encode_tokens.
+    def encode encoder, options = {}
+      if encoder.respond_to? :to_sym
+        CodeRay.encode(input, lang, encoder, options)
+      else
+        encoder.encode_tokens tokens, options
+      end
+    end
+    
+    # Tries to call encode;
+    # delegates to tokens otherwise.
+    def method_missing method, *args, &blk
+      encode method.to_sym, *args
+    rescue PluginHost::PluginNotFound
+      tokens.send(method, *args, &blk)
+    end
+    
+    # The (cached) result of the tokenized input; a Tokens instance.
+    def tokens
+      @tokens ||= scanner.tokenize(input)
+    end
+    
+    # A (cached) scanner instance to use for the scan task.
+    def scanner
+      @scanner ||= CodeRay.scanner(lang, options, &block)
+    end
+    
+    # Overwrite Struct#each.
+    def each *args, &blk
+      tokens.each(*args, &blk)
+      self
+    end
+    
+  end
+  
+end

data/lib/coderay/version.rb

@@ -0,0 +1,3 @@
+module CodeRay
+  VERSION = '1.0.0'
+end

data/test/functional/basic.rb

@@ -1,11 +1,15 @@
+# encoding: utf-8
 require 'test/unit'
+require File.expand_path('../../lib/assert_warning', __FILE__)
+
+$:.unshift File.expand_path('../../../lib', __FILE__)
 require 'coderay'
 
 class BasicTest < Test::Unit::TestCase
   
   def test_version
     assert_nothing_raised do
-      assert_match(/\A\d\.\d\.\d\z/, CodeRay::VERSION)
+      assert_match(/\A\d\.\d\.\d?\z/, CodeRay::VERSION)
     end
   end
   
@@ -14,36 +18,60 @@ class BasicTest < Test::Unit::TestCase
   RUBY_TEST_TOKENS = [
     ['puts', :ident],
     [' ', :space],
-    [:open, :string],
+    [:begin_group, :string],
       ['"', :delimiter],
       ['Hello, World!', :content],
       ['"', :delimiter],
-    [:close, :string]
-  ]
+    [:end_group, :string]
+  ].flatten
   def test_simple_scan
     assert_nothing_raised do
-      assert_equal RUBY_TEST_TOKENS, CodeRay.scan(RUBY_TEST_CODE, :ruby).to_ary
+      assert_equal RUBY_TEST_TOKENS, CodeRay.scan(RUBY_TEST_CODE, :ruby).tokens
     end
   end
   
-  RUBY_TEST_HTML = 'puts <span class="s"><span class="dl">&quot;</span>' + 
-    '<span class="k">Hello, World!</span><span class="dl">&quot;</span></span>'
+  RUBY_TEST_HTML = 'puts <span class="string"><span class="delimiter">&quot;</span>' + 
+    '<span class="content">Hello, World!</span><span class="delimiter">&quot;</span></span>'
   def test_simple_highlight
     assert_nothing_raised do
       assert_equal RUBY_TEST_HTML, CodeRay.scan(RUBY_TEST_CODE, :ruby).html
     end
   end
   
+  def test_scan_file
+    CodeRay.scan_file __FILE__
+  end
+  
+  def test_encode
+    assert_equal 1, CodeRay.encode('test', :python, :count)
+  end
+  
+  def test_encode_tokens
+    assert_equal 1, CodeRay.encode_tokens(CodeRay::Tokens['test', :string], :count)
+  end
+  
+  def test_encode_file
+    assert_equal File.read(__FILE__), CodeRay.encode_file(__FILE__, :text)
+  end
+  
+  def test_highlight
+    assert_match '<pre>test</pre>', CodeRay.highlight('test', :python)
+  end
+  
+  def test_highlight_file
+    assert_match "require <span class=\"string\"><span class=\"delimiter\">'</span><span class=\"content\">test/unit</span><span class=\"delimiter\">'</span></span>\n", CodeRay.highlight_file(__FILE__)
+  end
+  
   def test_duo
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE))
+      CodeRay::Duo[:plain, :text].highlight(RUBY_TEST_CODE))
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain => :plain].highlight(RUBY_TEST_CODE))
+      CodeRay::Duo[:plain => :text].highlight(RUBY_TEST_CODE))
   end
   
   def test_duo_stream
     assert_equal(RUBY_TEST_CODE,
-      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE, :stream => true))
+      CodeRay::Duo[:plain, :text].highlight(RUBY_TEST_CODE, :stream => true))
   end
   
   def test_comment_filter
@@ -98,25 +126,179 @@ more code  # and another comment, in-line.
     assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code
     assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code
     assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code
-    assert_equal 4, CodeRay.scan(rHTML, :rhtml).lines_of_code
+    assert_equal 4, CodeRay.scan(rHTML, :erb).lines_of_code
   end
   
-  def test_rubygems_not_loaded
-    assert_equal nil, defined? Gem
-  end if ENV['check_rubygems'] && RUBY_VERSION < '1.9'
-  
   def test_list_of_encoders
     assert_kind_of(Array, CodeRay::Encoders.list)
-    assert CodeRay::Encoders.list.include?('count')
+    assert CodeRay::Encoders.list.include?(:count)
   end
   
   def test_list_of_scanners
     assert_kind_of(Array, CodeRay::Scanners.list)
-    assert CodeRay::Scanners.list.include?('plaintext')
+    assert CodeRay::Scanners.list.include?(:text)
+  end
+  
+  def test_token_kinds
+    assert_kind_of Hash, CodeRay::TokenKinds
+    for kind, css_class in CodeRay::TokenKinds
+      assert_kind_of Symbol, kind
+      if css_class != false
+        assert_kind_of String, css_class, "TokenKinds[%p] == %p" % [kind, css_class]
+      end
+    end
+    assert_equal 'reserved', CodeRay::TokenKinds[:reserved]
+    assert_warning 'Undefined Token kind: :shibboleet' do
+      assert_equal false, CodeRay::TokenKinds[:shibboleet]
+    end
+  end
+  
+  class Milk < CodeRay::Encoders::Encoder
+    FILE_EXTENSION = 'cocoa'
+  end
+  
+  class HoneyBee < CodeRay::Encoders::Encoder
+  end
+  
+  def test_encoder_file_extension
+    assert_nothing_raised do
+      assert_equal 'html', CodeRay::Encoders::Page::FILE_EXTENSION
+      assert_equal 'cocoa', Milk::FILE_EXTENSION
+      assert_equal 'cocoa', Milk.new.file_extension
+      assert_equal 'honeybee', HoneyBee::FILE_EXTENSION
+      assert_equal 'honeybee', HoneyBee.new.file_extension
+    end
+    assert_raise NameError do
+      HoneyBee::MISSING_CONSTANT
+    end
+  end
+  
+  def test_encoder_tokens
+    encoder = CodeRay::Encoders::Encoder.new
+    encoder.send :setup, {}
+    assert_raise(ArgumentError) { encoder.token :strange, '' }
+    encoder.token 'test', :debug
+  end
+  
+  def test_encoder_deprecated_interface
+    encoder = CodeRay::Encoders::Encoder.new
+    encoder.send :setup, {}
+    assert_warning 'Using old Tokens#<< interface.' do
+      encoder << ['test', :content]
+    end
+    assert_raise ArgumentError do
+      encoder << [:strange, :input]
+    end
+    assert_raise ArgumentError do
+      encoder.encode_tokens [['test', :token]]
+    end
+  end
+  
+  def encoder_token_interface_deprecation_warning_given
+    CodeRay::Encoders::Encoder.send :class_variable_get, :@@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
+  end
+  
+  def test_scanner_file_extension
+    assert_equal 'rb', CodeRay::Scanners::Ruby.file_extension
+    assert_equal 'rb', CodeRay::Scanners::Ruby.new.file_extension
+    assert_equal 'java', CodeRay::Scanners::Java.file_extension
+    assert_equal 'java', CodeRay::Scanners::Java.new.file_extension
+  end
+  
+  def test_scanner_lang
+    assert_equal :ruby, CodeRay::Scanners::Ruby.lang
+    assert_equal :ruby, CodeRay::Scanners::Ruby.new.lang
+    assert_equal :java, CodeRay::Scanners::Java.lang
+    assert_equal :java, CodeRay::Scanners::Java.new.lang
+  end
+  
+  def test_scanner_tokenize
+    assert_equal ['foo', :plain], CodeRay::Scanners::Plain.new.tokenize('foo')
+    assert_equal [['foo', :plain], ['bar', :plain]], CodeRay::Scanners::Plain.new.tokenize(['foo', 'bar'])
+    CodeRay::Scanners::Plain.new.tokenize 42
+  end
+  
+  def test_scanner_tokens
+    scanner = CodeRay::Scanners::Plain.new
+    scanner.tokenize('foo')
+    assert_equal ['foo', :plain], scanner.tokens
+    scanner.string = ''
+    assert_equal ['', :plain], scanner.tokens
+  end
+  
+  def test_scanner_line_and_column
+    scanner = CodeRay::Scanners::Plain.new "foo\nbär+quux"
+    assert_equal 0, scanner.pos
+    assert_equal 1, scanner.line
+    assert_equal 1, scanner.column
+    scanner.scan(/foo/)
+    assert_equal 3, scanner.pos
+    assert_equal 1, scanner.line
+    assert_equal 4, scanner.column
+    scanner.scan(/\n/)
+    assert_equal 4, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 1, scanner.column
+    scanner.scan(/b/)
+    assert_equal 5, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 2, scanner.column
+    scanner.scan(/a/)
+    assert_equal 5, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 2, scanner.column
+    scanner.scan(/ä/)
+    assert_equal 7, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 4, scanner.column
+    scanner.scan(/r/)
+    assert_equal 8, scanner.pos
+    assert_equal 2, scanner.line
+    assert_equal 5, scanner.column
+  end
+  
+  def test_scanner_use_subclasses
+    assert_raise NotImplementedError do
+      CodeRay::Scanners::Scanner.new
+    end
+  end
+  
+  class InvalidScanner < CodeRay::Scanners::Scanner
+  end
+  
+  def test_scanner_scan_tokens
+    assert_raise NotImplementedError do
+      InvalidScanner.new.tokenize ''
+    end
+  end
+  
+  class RaisingScanner < CodeRay::Scanners::Scanner
+    def scan_tokens encoder, options
+      raise_inspect 'message', [], :initial
+    end
+  end
+  
+  def test_scanner_raise_inspect
+    assert_raise CodeRay::Scanners::Scanner::ScanError do
+      RaisingScanner.new.tokenize ''
+    end
   end
   
   def test_scan_a_frozen_string
-    CodeRay.scan RUBY_VERSION, :ruby
+    assert_nothing_raised do
+      CodeRay.scan RUBY_VERSION, :ruby
+      CodeRay.scan RUBY_VERSION, :plain
+    end
+  end
+  
+  def test_scan_a_non_string
+    assert_nothing_raised do
+      CodeRay.scan 42, :ruby
+      CodeRay.scan nil, :ruby
+      CodeRay.scan self, :ruby
+      CodeRay.encode ENV.to_hash, :ruby, :page
+      CodeRay.highlight CodeRay, :plain
+    end
   end
   
 end

data/test/functional/examples.rb

@@ -0,0 +1,130 @@
+require 'test/unit'
+
+$:.unshift File.expand_path('../../../lib', __FILE__)
+require 'coderay'
+
+class ExamplesTest < Test::Unit::TestCase
+  
+  def test_examples
+    # output as HTML div (using inline CSS styles)
+    div = CodeRay.scan('puts "Hello, world!"', :ruby).div
+    assert_equal <<-DIV, div
+<div class="CodeRay">
+  <div class="code"><pre>puts <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">Hello, world!</span><span style="color:#710">&quot;</span></span></pre></div>
+</div>
+    DIV
+    
+    # ...with line numbers
+    div = CodeRay.scan(<<-CODE.chomp, :ruby).div(:line_numbers => :table)
+5.times do
+  puts 'Hello, world!'
+end
+    CODE
+    assert_equal <<-DIV, div
+<table class="CodeRay"><tr>
+  <td class="line-numbers" title="double click to toggle" ondblclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><a href="#n1" name="n1">1</a>
+<a href="#n2" name="n2">2</a>
+<a href="#n3" name="n3">3</a>
+</pre></td>
+  <td class="code"><pre><span style="color:#00D">5</span>.times <span style="color:#080;font-weight:bold">do</span>
+  puts <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">'</span><span style="color:#D20">Hello, world!</span><span style="color:#710">'</span></span>
+<span style="color:#080;font-weight:bold">end</span></pre></td>
+</tr></table>
+    DIV
+    
+    # output as standalone HTML page (using CSS classes)
+    page = CodeRay.scan('puts "Hello, world!"', :ruby).page
+    assert page[<<-PAGE]
+<body style="background-color: white;">
+
+<table class="CodeRay"><tr>
+  <td class="line-numbers" title="double click to toggle" ondblclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre>
+</pre></td>
+  <td class="code"><pre>puts <span class="string"><span class="delimiter">&quot;</span><span class="content">Hello, world!</span><span class="delimiter">&quot;</span></span></pre></td>
+</tr></table>
+
+</body>
+    PAGE
+    
+    # keep scanned tokens for later use
+    tokens = CodeRay.scan('{ "just": "an", "example": 42 }', :json)
+    assert_kind_of CodeRay::TokensProxy, tokens
+    
+    assert_equal ["{", :operator, " ", :space, :begin_group, :key,
+      "\"", :delimiter, "just", :content, "\"", :delimiter,
+      :end_group, :key, ":", :operator, " ", :space,
+      :begin_group, :string, "\"", :delimiter, "an", :content,
+      "\"", :delimiter, :end_group, :string, ",", :operator,
+      " ", :space, :begin_group, :key, "\"", :delimiter,
+      "example", :content, "\"", :delimiter, :end_group, :key,
+      ":", :operator, " ", :space, "42", :integer,
+      " ", :space, "}", :operator], tokens.tokens
+    
+    # produce a token statistic
+    assert_equal <<-STATISTIC, tokens.statistic
+
+Code Statistics
+
+Tokens                  26
+  Non-Whitespace        15
+Bytes Total             31
+
+Token Types (7):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+  TOTAL                       26  100.00 %     1.2
+  delimiter                    6   23.08 %     1.0
+  operator                     5   19.23 %     1.0
+  space                        5   19.23 %     1.0
+  key                          4   15.38 %     0.0
+  :begin_group                 3   11.54 %     0.0
+  :end_group                   3   11.54 %     0.0
+  content                      3   11.54 %     4.3
+  string                       2    7.69 %     0.0
+  integer                      1    3.85 %     2.0
+
+    STATISTIC
+    
+    # count the tokens
+    assert_equal 26, tokens.count
+    
+    # produce a HTML div, but with CSS classes
+    div = tokens.div(:css => :class)
+    assert_equal <<-DIV, div
+<div class="CodeRay">
+  <div class="code"><pre>{ <span class="key"><span class="delimiter">&quot;</span><span class="content">just</span><span class="delimiter">&quot;</span></span>: <span class="string"><span class="delimiter">&quot;</span><span class="content">an</span><span class="delimiter">&quot;</span></span>, <span class="key"><span class="delimiter">&quot;</span><span class="content">example</span><span class="delimiter">&quot;</span></span>: <span class="integer">42</span> }</pre></div>
+</div>
+    DIV
+    
+    # highlight a file (HTML div); guess the file type base on the extension
+    require 'coderay/helpers/file_type'
+    assert_equal :ruby, CodeRay::FileType[__FILE__]
+    
+    # get a new scanner for Python
+    python_scanner = CodeRay.scanner :python
+    assert_kind_of CodeRay::Scanners::Python, python_scanner
+    
+    # get a new encoder for terminal
+    terminal_encoder = CodeRay.encoder :term
+    assert_kind_of CodeRay::Encoders::Terminal, terminal_encoder
+    
+    # scanning into tokens
+    tokens = python_scanner.tokenize 'import this;  # The Zen of Python'
+    assert_equal ["import", :keyword, " ", :space, "this", :include,
+      ";", :operator, "  ", :space, "# The Zen of Python", :comment], tokens
+    
+    # format the tokens
+    term = terminal_encoder.encode_tokens(tokens)
+    assert_equal "\e[1;31mimport\e[0m \e[33mthis\e[0m;  \e[37m# The Zen of Python\e[0m", term
+    
+    # re-using scanner and encoder
+    ruby_highlighter = CodeRay::Duo[:ruby, :div]
+    div = ruby_highlighter.encode('puts "Hello, world!"')
+    assert_equal <<-DIV, div
+<div class="CodeRay">
+  <div class="code"><pre>puts <span style="background-color:hsla(0,100%,50%,0.05)"><span style="color:#710">&quot;</span><span style="color:#D20">Hello, world!</span><span style="color:#710">&quot;</span></span></pre></div>
+</div>
+    DIV
+  end
+  
+end