coderay 0.9.1 → 0.9.2

data/lib/coderay/scanners/ruby.rb

@@ -284,14 +284,23 @@ module Scanners
               kind = :class_variable
 
             else
-              kind = :error
-              match = (scan(/./mu) rescue nil) || getch
-              if !unicode && match.size > 1
-                # warn 'Switching to unicode mode because of char %p' % [match]
-                unicode = true
-                unscan
-                next
+              if !unicode
+                # check for unicode
+                debug, $DEBUG = $DEBUG, false
+                begin
+                  if check(/./mu).size > 1
+                    # seems like we should try again with unicode
+                    unicode = true
+                  end
+                rescue
+                  # bad unicode char; use getch
+                ensure
+                  $DEBUG = debug
+                end
+                next if unicode
               end
+              kind = :error
+              match = getch
 
             end
 
@@ -370,7 +379,7 @@ module Scanners
             last_token_dot = last_token_dot == :set
           end
           
-          if $DEBUG and not kind
+          if $CODERAY_DEBUG and not kind
             raise_inspect 'Error token %p in line %d' %
               [[match, kind], line], tokens, state
           end

data/lib/coderay/scanners/scheme.rb

@@ -123,7 +123,7 @@ module CodeRay
           end
           
           match ||= matched
-          if $DEBUG and not kind
+          if $CODERAY_DEBUG and not kind
             raise_inspect 'Error token %p in line %d' %
             [[match, kind], line], tokens
           end

data/lib/coderay/scanners/yaml.rb

@@ -21,14 +21,7 @@ module Scanners
         
         kind = nil
         match = nil
-        
-        if bol?
-          key_indent = nil
-          if $DEBUG
-            indent = check(/ +/) ? matched.size : 0
-            tokens << [indent.to_s, :debug]
-          end
-        end
+        key_indent = nil if bol?
         
         if match = scan(/ +[\t ]*/)
           kind = :space
@@ -128,8 +121,11 @@ module Scanners
         
         match ||= matched
         
-        raise_inspect 'Error token %p in line %d' % [[match, kind], line], tokens if $DEBUG && !kind
-        raise_inspect 'Empty token', tokens unless match
+        if $CODERAY_DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens, state unless match
         
         tokens << [match, kind]
         

data/lib/coderay/tokens.rb

@@ -129,7 +129,6 @@ module CodeRay
     # for example, consecutive //-comment lines could already be
     # joined in one comment token by the Scanner.
     def optimize
-      print ' Tokens#optimize: before: %d - ' % size if $DEBUG
       last_kind = last_text = nil
       new = self.class.new
       for text, kind in self
@@ -148,8 +147,6 @@ module CodeRay
         end
       end
       new << [last_text, last_kind] if last_kind
-      print 'after: %d (%d saved = %2.0f%%)' %
-        [new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
       new
     end
 

data/test/functional/basic.rb

@@ -0,0 +1,118 @@
+require 'test/unit'
+require 'coderay'
+
+class BasicTest < Test::Unit::TestCase
+  
+  def test_version
+    assert_nothing_raised do
+      assert_match(/\A\d\.\d\.\d\z/, CodeRay::VERSION)
+    end
+  end
+  
+  RUBY_TEST_CODE = 'puts "Hello, World!"'
+  
+  RUBY_TEST_TOKENS = [
+    ['puts', :ident],
+    [' ', :space],
+    [:open, :string],
+      ['"', :delimiter],
+      ['Hello, World!', :content],
+      ['"', :delimiter],
+    [:close, :string]
+  ]
+  def test_simple_scan
+    assert_nothing_raised do
+      assert_equal RUBY_TEST_TOKENS, CodeRay.scan(RUBY_TEST_CODE, :ruby).to_ary
+    end
+  end
+  
+  RUBY_TEST_HTML = 'puts <span class="s"><span class="dl">&quot;</span>' + 
+    '<span class="k">Hello, World!</span><span class="dl">&quot;</span></span>'
+  def test_simple_highlight
+    assert_nothing_raised do
+      assert_equal RUBY_TEST_HTML, CodeRay.scan(RUBY_TEST_CODE, :ruby).html
+    end
+  end
+  
+  def test_duo
+    assert_equal(RUBY_TEST_CODE,
+      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE))
+    assert_equal(RUBY_TEST_CODE,
+      CodeRay::Duo[:plain => :plain].highlight(RUBY_TEST_CODE))
+  end
+  
+  def test_duo_stream
+    assert_equal(RUBY_TEST_CODE,
+      CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE, :stream => true))
+  end
+  
+  def test_comment_filter
+    assert_equal <<-EXPECTED, CodeRay.scan(<<-INPUT, :ruby).comment_filter.text
+#!/usr/bin/env ruby
+
+code
+
+more code  
+      EXPECTED
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code  # and another comment, in-line.
+      INPUT
+  end
+  
+  def test_lines_of_code
+    assert_equal 2, CodeRay.scan(<<-INPUT, :ruby).lines_of_code
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code  # and another comment, in-line.
+      INPUT
+    rHTML = <<-RHTML
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+  <title><%= controller.controller_name.titleize %>: <%= controller.action_name %></title>
+  <%= stylesheet_link_tag 'scaffold' %>
+</head>
+<body>
+
+<p style="color: green"><%= flash[:notice] %></p>
+
+<div id="main">
+  <%= yield %>
+</div>
+
+</body>
+</html>
+      RHTML
+    assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code
+    assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code
+    assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code
+    assert_equal 4, CodeRay.scan(rHTML, :rhtml).lines_of_code
+  end
+  
+  def test_rubygems_not_loaded
+    assert_equal nil, defined? Gem
+  end if ENV['check_rubygems'] && RUBY_VERSION < '1.9'
+  
+  def test_list_of_encoders
+    assert_kind_of(Array, CodeRay::Encoders.list)
+    assert CodeRay::Encoders.list.include?('count')
+  end
+  
+  def test_list_of_scanners
+    assert_kind_of(Array, CodeRay::Scanners.list)
+    assert CodeRay::Scanners.list.include?('plaintext')
+  end
+  
+end

data/test/functional/for_redcloth.rb

@@ -0,0 +1,77 @@
+require 'test/unit'
+$: << 'lib'
+require 'coderay'
+
+begin
+  require 'rubygems' unless defined? Gem
+  gem 'RedCloth', '>= 4.0.3' rescue nil
+  require 'redcloth'
+rescue LoadError
+  warn 'RedCloth not found - skipping for_redcloth tests.'
+end
+
+class BasicTest < Test::Unit::TestCase
+  
+  def test_for_redcloth
+    require 'coderay/for_redcloth'
+    assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">puts <span style=\"background-color:#fff0f0;color:#D20\"><span style=\"color:#710\">&quot;</span><span style=\"\">Hello, World!</span><span style=\"color:#710\">&quot;</span></span></span></p>",
+      RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
+    assert_equal <<-BLOCKCODE.chomp,
+<div lang="ruby" class="CodeRay">
+  <div class="code"><pre>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">&quot;</span><span style="">Hello, World!</span><span style="color:#710">&quot;</span></span></pre></div>
+</div>
+      BLOCKCODE
+      RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html
+  end
+  
+  def test_for_redcloth_no_lang
+    require 'coderay/for_redcloth'
+    assert_equal "<p><code>puts \"Hello, World!\"</code></p>",
+      RedCloth.new('@puts "Hello, World!"@').to_html
+    assert_equal <<-BLOCKCODE.chomp,
+<pre><code>puts \"Hello, World!\"</code></pre>
+      BLOCKCODE
+      RedCloth.new('bc. puts "Hello, World!"').to_html
+  end
+  
+  def test_for_redcloth_style
+    require 'coderay/for_redcloth'
+    assert_equal <<-BLOCKCODE.chomp,
+<pre style=\"color: red;\"><code style=\"color: red;\">puts \"Hello, World!\"</code></pre>
+      BLOCKCODE
+      RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html
+  end
+  
+  def test_for_redcloth_escapes
+    require 'coderay/for_redcloth'
+    assert_equal '<p><span lang="ruby" class="CodeRay">&gt;</span></p>',
+      RedCloth.new('@[ruby]>@').to_html
+    assert_equal <<-BLOCKCODE.chomp,
+<div lang="ruby" class="CodeRay">
+  <div class="code"><pre>&amp;</pre></div>
+</div>
+      BLOCKCODE
+      RedCloth.new('bc[ruby]. &').to_html
+  end
+  
+  def test_for_redcloth_escapes2
+    require 'coderay/for_redcloth'
+    assert_equal "<p><span lang=\"c\" class=\"CodeRay\"><span style=\"color:#579\">#include</span> <span style=\"color:#B44;font-weight:bold\">&lt;test.h&gt;</span></span></p>",
+      RedCloth.new('@[c]#include <test.h>@').to_html
+  end
+  
+  # See http://jgarber.lighthouseapp.com/projects/13054/tickets/124-code-markup-does-not-allow-brackets.
+  def test_for_redcloth_false_positive
+    require 'coderay/for_redcloth'
+    assert_equal '<p><code>[project]_dff.skjd</code></p>',
+      RedCloth.new('@[project]_dff.skjd@').to_html
+    # false positive, but expected behavior / known issue
+    assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">_dff.skjd</span></p>",
+      RedCloth.new('@[ruby]_dff.skjd@').to_html
+    assert_equal <<-BLOCKCODE.chomp,
+<pre><code>[project]_dff.skjd</code></pre>
+      BLOCKCODE
+      RedCloth.new('bc. [project]_dff.skjd').to_html
+  end
+  
+end if defined? RedCloth

data/test/functional/load_plugin_scanner.rb

@@ -0,0 +1,11 @@
+require 'test/unit'
+require 'coderay'
+
+class PluginScannerTest < Test::Unit::TestCase
+  
+  def test_load
+    require File.join(File.dirname(__FILE__), 'vhdl')
+    assert_equal 'VHDL', CodeRay.scanner(:vhdl).class.name
+  end
+  
+end

data/test/functional/suite.rb

@@ -0,0 +1,12 @@
+require 'test/unit'
+
+MYDIR = File.dirname(__FILE__)
+
+$: << 'lib'
+require 'coderay'
+puts "Running basic CodeRay #{CodeRay::VERSION} tests..."
+
+suite = %w(basic load_plugin_scanner word_list)
+for test_case in suite
+  load File.join(MYDIR, test_case + '.rb')
+end

data/test/functional/vhdl.rb

@@ -0,0 +1,126 @@
+class VHDL < CodeRay::Scanners::Scanner
+
+  register_for :vhdl
+
+  RESERVED_WORDS = [
+    'access','after','alias','all','assert','architecture','begin',
+    'block','body','buffer','bus','case','component','configuration','constant',
+    'disconnect','downto','else','elsif','end','entity','exit','file','for',
+    'function','generate','generic','group','guarded','if','impure','in',
+    'inertial','inout','is','label','library','linkage','literal','loop',
+    'map','new','next','null','of','on','open','others','out','package',
+    'port','postponed','procedure','process','pure','range','record','register',
+    'reject','report','return','select','severity','signal','shared','subtype',
+    'then','to','transport','type','unaffected','units','until','use','variable',
+    'wait','when','while','with','note','warning','error','failure','and',
+    'or','xor','not','nor',
+    'array'
+  ]
+
+  PREDEFINED_TYPES = [
+    'bit','bit_vector','character','boolean','integer','real','time','string',
+    'severity_level','positive','natural','signed','unsigned','line','text',
+    'std_logic','std_logic_vector','std_ulogic','std_ulogic_vector','qsim_state',
+    'qsim_state_vector','qsim_12state','qsim_12state_vector','qsim_strength',
+    'mux_bit','mux_vector','reg_bit','reg_vector','wor_bit','wor_vector'
+  ]
+
+  PREDEFINED_CONSTANTS = [
+
+  ]
+
+  IDENT_KIND = CodeRay::CaseIgnoringWordList.new(:ident).
+    add(RESERVED_WORDS, :reserved).
+    add(PREDEFINED_TYPES, :pre_type).
+    add(PREDEFINED_CONSTANTS, :pre_constant)
+
+  ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+  UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+  def scan_tokens tokens, options
+
+    state = :initial
+
+    until eos?
+
+      kind = nil
+      match = nil
+
+      case state
+
+      when :initial
+
+        if scan(/ \s+ | \\\n /x)
+          kind = :space
+
+        elsif scan(/-- .*/x)
+          kind = :comment
+
+        elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
+          kind = :operator
+
+        elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+          kind = IDENT_KIND[match.downcase]
+
+        elsif match = scan(/[a-z]?"/i)
+          tokens << [:open, :string]
+          state = :string
+          kind = :delimiter
+
+        elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+          kind = :char
+
+        elsif scan(/(?:\d+)(?![.eEfF])/)
+          kind = :integer
+
+        elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+          kind = :float
+
+        else
+          getch
+          kind = :error
+
+        end
+
+      when :string
+        if scan(/[^\\\n"]+/)
+          kind = :content
+        elsif scan(/"/)
+          tokens << ['"', :delimiter]
+          tokens << [:close, :string]
+          state = :initial
+          next
+        elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+          kind = :char
+        elsif scan(/ \\ | $ /x)
+          tokens << [:close, :string]
+          kind = :error
+          state = :initial
+        else
+          raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+        end
+
+      else
+        raise_inspect 'Unknown state', tokens
+
+      end
+
+      match ||= matched
+      if $DEBUG and not kind
+        raise_inspect 'Error token %p in line %d' %
+          [[match, kind], line], tokens
+      end
+      raise_inspect 'Empty token', tokens unless match
+
+      tokens << [match, kind]
+
+    end
+
+    if state == :string
+      tokens << [:close, :string]
+    end
+
+    tokens
+  end
+
+end