coderay 0.9.6.pre → 0.9.6.pre2

data/lib/coderay/scanner.rb

@@ -69,6 +69,7 @@ module CodeRay
         def normify code
           code = code.to_s
           if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
+            code = code.dup
             original_encoding = code.encoding
             code.force_encoding 'Windows-1252'
             unless code.valid_encoding?

data/test/functional/basic.rbc

@@ -0,0 +1,1938 @@
+!RBIX
+0
+x
+M
+1
+n
+n
+x
+10
+__script__
+i
+53
+5
+7
+0
+64
+47
+49
+1
+1
+15
+5
+7
+2
+64
+47
+49
+1
+1
+15
+99
+7
+3
+45
+4
+5
+43
+6
+43
+7
+65
+49
+8
+3
+13
+99
+12
+7
+9
+12
+7
+10
+12
+65
+12
+49
+11
+4
+15
+49
+9
+0
+15
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+n
+p
+12
+s
+9
+test/unit
+x
+7
+require
+s
+7
+coderay
+x
+9
+BasicTest
+x
+4
+Test
+n
+x
+4
+Unit
+x
+8
+TestCase
+x
+10
+open_class
+x
+14
+__class_init__
+M
+1
+n
+n
+x
+9
+BasicTest
+i
+249
+5
+66
+99
+7
+0
+7
+1
+65
+67
+49
+2
+0
+49
+3
+4
+15
+65
+7
+4
+7
+5
+64
+49
+6
+2
+15
+65
+7
+7
+7
+8
+64
+7
+9
+35
+2
+7
+10
+64
+7
+11
+35
+2
+7
+12
+7
+13
+35
+2
+7
+14
+64
+7
+15
+35
+2
+7
+16
+64
+7
+17
+35
+2
+7
+14
+64
+7
+15
+35
+2
+7
+18
+7
+13
+35
+2
+35
+7
+49
+6
+2
+15
+99
+7
+19
+7
+20
+65
+67
+49
+2
+0
+49
+3
+4
+15
+65
+7
+21
+7
+22
+64
+7
+23
+64
+81
+24
+49
+6
+2
+15
+99
+7
+25
+7
+26
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+27
+7
+28
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+29
+7
+30
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+31
+7
+32
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+33
+7
+34
+65
+67
+49
+2
+0
+49
+3
+4
+15
+45
+35
+36
+7
+37
+64
+49
+38
+1
+13
+9
+202
+15
+45
+39
+40
+7
+41
+64
+84
+42
+9
+219
+99
+7
+43
+7
+44
+65
+67
+49
+2
+0
+49
+3
+4
+8
+220
+1
+15
+99
+7
+45
+7
+46
+65
+67
+49
+2
+0
+49
+3
+4
+15
+99
+7
+47
+7
+48
+65
+67
+49
+2
+0
+49
+3
+4
+11
+I
+a
+I
+0
+I
+0
+I
+0
+n
+p
+49
+x
+12
+test_version
+M
+1
+n
+n
+x
+12
+test_version
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+12
+test_version
+i
+29
+5
+7
+0
+13
+70
+9
+19
+15
+44
+43
+1
+7
+2
+78
+49
+3
+2
+6
+0
+45
+4
+5
+43
+6
+47
+49
+7
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+8
+n
+x
+6
+Regexp
+s
+14
+\A\d\.\d\.\d\z
+x
+3
+new
+x
+7
+CodeRay
+n
+x
+7
+VERSION
+x
+12
+assert_match
+p
+5
+I
+0
+I
+7
+I
+0
+I
+8
+I
+1d
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+6
+I
+0
+I
+7
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+17
+method_visibility
+x
+15
+add_defn_method
+x
+14
+RUBY_TEST_CODE
+s
+20
+puts "Hello, World!"
+x
+9
+const_set
+x
+16
+RUBY_TEST_TOKENS
+s
+4
+puts
+x
+5
+ident
+s
+1
+ 
+x
+5
+space
+x
+4
+open
+x
+6
+string
+s
+1
+"
+x
+9
+delimiter
+s
+13
+Hello, World!
+x
+7
+content
+x
+5
+close
+x
+16
+test_simple_scan
+M
+1
+n
+n
+x
+16
+test_simple_scan
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+16
+test_simple_scan
+i
+23
+5
+45
+0
+1
+45
+2
+3
+45
+4
+5
+7
+6
+49
+7
+2
+49
+8
+0
+47
+49
+9
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+10
+x
+16
+RUBY_TEST_TOKENS
+n
+x
+7
+CodeRay
+n
+x
+14
+RUBY_TEST_CODE
+n
+x
+4
+ruby
+x
+4
+scan
+x
+6
+to_ary
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+18
+I
+0
+I
+19
+I
+17
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+17
+I
+0
+I
+18
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+14
+RUBY_TEST_HTML
+s
+51
+puts <span class="s"><span class="dl">&quot;</span>
+s
+73
+<span class="k">Hello, World!</span><span class="dl">&quot;</span></span>
+x
+1
++
+x
+21
+test_simple_highlight
+M
+1
+n
+n
+x
+21
+test_simple_highlight
+i
+8
+5
+56
+0
+47
+50
+1
+0
+11
+I
+2
+I
+0
+I
+0
+I
+0
+n
+p
+2
+M
+1
+p
+2
+x
+9
+for_block
+t
+n
+x
+21
+test_simple_highlight
+i
+23
+5
+45
+0
+1
+45
+2
+3
+45
+4
+5
+7
+6
+49
+7
+2
+49
+8
+0
+47
+49
+9
+2
+11
+I
+6
+I
+0
+I
+0
+I
+0
+I
+-2
+p
+10
+x
+14
+RUBY_TEST_HTML
+n
+x
+7
+CodeRay
+n
+x
+14
+RUBY_TEST_CODE
+n
+x
+4
+ruby
+x
+4
+scan
+x
+4
+html
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+20
+I
+0
+I
+21
+I
+17
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+assert_nothing_raised
+p
+5
+I
+0
+I
+1f
+I
+0
+I
+20
+I
+8
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+8
+test_duo
+M
+1
+n
+n
+x
+8
+test_duo
+i
+66
+5
+45
+0
+1
+45
+2
+3
+43
+4
+7
+5
+7
+5
+49
+6
+2
+45
+0
+7
+49
+8
+1
+47
+49
+9
+2
+15
+5
+45
+0
+10
+45
+2
+11
+43
+4
+44
+43
+12
+79
+49
+13
+1
+13
+7
+5
+7
+5
+49
+14
+2
+15
+49
+6
+1
+45
+0
+15
+49
+8
+1
+47
+49
+9
+2
+11
+I
+7
+I
+0
+I
+0
+I
+0
+n
+p
+16
+x
+14
+RUBY_TEST_CODE
+n
+x
+7
+CodeRay
+n
+x
+3
+Duo
+x
+5
+plain
+x
+2
+[]
+n
+x
+9
+highlight
+x
+12
+assert_equal
+n
+n
+x
+4
+Hash
+x
+16
+new_from_literal
+x
+3
+[]=
+n
+p
+15
+I
+0
+I
+25
+I
+0
+I
+26
+I
+4
+I
+27
+I
+16
+I
+26
+I
+1b
+I
+28
+I
+1f
+I
+29
+I
+3d
+I
+28
+I
+42
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+15
+test_duo_stream
+M
+1
+n
+n
+x
+15
+test_duo_stream
+i
+42
+5
+45
+0
+1
+45
+2
+3
+43
+4
+7
+5
+7
+5
+49
+6
+2
+45
+0
+7
+44
+43
+8
+79
+49
+9
+1
+13
+7
+10
+2
+49
+11
+2
+15
+49
+12
+2
+47
+49
+13
+2
+11
+I
+8
+I
+0
+I
+0
+I
+0
+n
+p
+14
+x
+14
+RUBY_TEST_CODE
+n
+x
+7
+CodeRay
+n
+x
+3
+Duo
+x
+5
+plain
+x
+2
+[]
+n
+x
+4
+Hash
+x
+16
+new_from_literal
+x
+6
+stream
+x
+3
+[]=
+x
+9
+highlight
+x
+12
+assert_equal
+p
+9
+I
+0
+I
+2c
+I
+0
+I
+2d
+I
+4
+I
+2e
+I
+25
+I
+2d
+I
+2a
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+19
+test_comment_filter
+M
+1
+n
+n
+x
+19
+test_comment_filter
+i
+26
+5
+7
+0
+64
+45
+1
+2
+7
+3
+64
+7
+4
+49
+5
+2
+49
+6
+0
+49
+7
+0
+47
+49
+8
+2
+11
+I
+5
+I
+0
+I
+0
+I
+0
+n
+p
+9
+s
+39
+#!/usr/bin/env ruby
+
+code
+
+more code  
+
+x
+7
+CodeRay
+n
+s
+127
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code  # and another comment, in-line.
+
+x
+4
+ruby
+x
+4
+scan
+x
+14
+comment_filter
+x
+4
+text
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+31
+I
+0
+I
+32
+I
+1a
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+18
+test_lines_of_code
+M
+1
+n
+n
+x
+18
+test_lines_of_code
+i
+108
+5
+80
+45
+0
+1
+7
+2
+64
+7
+3
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+7
+7
+64
+19
+0
+15
+5
+78
+45
+0
+8
+20
+0
+7
+9
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+78
+45
+0
+10
+20
+0
+7
+11
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+78
+45
+0
+12
+20
+0
+7
+13
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+15
+5
+4
+4
+45
+0
+14
+20
+0
+7
+15
+49
+4
+2
+49
+5
+0
+47
+49
+6
+2
+11
+I
+6
+I
+1
+I
+0
+I
+0
+n
+p
+16
+x
+7
+CodeRay
+n
+s
+127
+#!/usr/bin/env ruby
+=begin
+A multi-line comment.
+=end
+code
+# A single-line comment.
+more code  # and another comment, in-line.
+
+x
+4
+ruby
+x
+4
+scan
+x
+13
+lines_of_code
+x
+12
+assert_equal
+s
+514
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+  <meta http-equiv="content-type" content="text/html;charset=UTF-8" />
+  <title><%= controller.controller_name.titleize %>: <%= controller.action_name %></title>
+  <%= stylesheet_link_tag 'scaffold' %>
+</head>
+<body>
+
+<p style="color: green"><%= flash[:notice] %></p>
+
+<div id="main">
+  <%= yield %>
+</div>
+
+</body>
+</html>
+
+n
+x
+4
+html
+n
+x
+3
+php
+n
+x
+4
+yaml
+n
+x
+5
+rhtml
+p
+15
+I
+0
+I
+43
+I
+0
+I
+44
+I
+15
+I
+4d
+I
+1b
+I
+62
+I
+2f
+I
+63
+I
+43
+I
+64
+I
+57
+I
+65
+I
+6c
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+1
+x
+5
+rHTML
+x
+3
+ENV
+n
+s
+14
+check_rubygems
+x
+2
+[]
+x
+12
+RUBY_VERSION
+n
+s
+3
+1.9
+x
+1
+<
+x
+24
+test_rubygems_not_loaded
+M
+1
+n
+n
+x
+24
+test_rubygems_not_loaded
+i
+34
+5
+1
+26
+93
+0
+15
+29
+17
+0
+7
+0
+98
+1
+1
+30
+8
+23
+25
+92
+0
+27
+8
+28
+15
+7
+2
+8
+29
+1
+47
+49
+3
+2
+11
+I
+4
+I
+0
+I
+0
+I
+0
+n
+p
+4
+x
+3
+Gem
+x
+16
+vm_const_defined
+s
+8
+constant
+x
+12
+assert_equal
+p
+5
+I
+0
+I
+68
+I
+0
+I
+69
+I
+22
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+test_list_of_encoders
+M
+1
+n
+n
+x
+21
+test_list_of_encoders
+i
+37
+5
+45
+0
+1
+45
+2
+3
+43
+4
+49
+5
+0
+47
+49
+6
+2
+15
+5
+45
+2
+7
+43
+4
+49
+5
+0
+7
+8
+64
+49
+9
+1
+47
+49
+10
+1
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+11
+x
+5
+Array
+n
+x
+7
+CodeRay
+n
+x
+8
+Encoders
+x
+4
+list
+x
+14
+assert_kind_of
+n
+s
+5
+count
+x
+8
+include?
+x
+6
+assert
+p
+7
+I
+0
+I
+6c
+I
+0
+I
+6d
+I
+11
+I
+6e
+I
+25
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+21
+test_list_of_scanners
+M
+1
+n
+n
+x
+21
+test_list_of_scanners
+i
+37
+5
+45
+0
+1
+45
+2
+3
+43
+4
+49
+5
+0
+47
+49
+6
+2
+15
+5
+45
+2
+7
+43
+4
+49
+5
+0
+7
+8
+64
+49
+9
+1
+47
+49
+10
+1
+11
+I
+3
+I
+0
+I
+0
+I
+0
+n
+p
+11
+x
+5
+Array
+n
+x
+7
+CodeRay
+n
+x
+8
+Scanners
+x
+4
+list
+x
+14
+assert_kind_of
+n
+s
+9
+plaintext
+x
+8
+include?
+x
+6
+assert
+p
+7
+I
+0
+I
+71
+I
+0
+I
+72
+I
+11
+I
+73
+I
+25
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+p
+49
+I
+2
+I
+6
+I
+10
+I
+c
+I
+1a
+I
+e
+I
+1d
+I
+f
+I
+24
+I
+10
+I
+2b
+I
+11
+I
+31
+I
+12
+I
+38
+I
+13
+I
+3f
+I
+14
+I
+46
+I
+15
+I
+52
+I
+17
+I
+60
+I
+1d
+I
+63
+I
+1d
+I
+66
+I
+1e
+I
+6f
+I
+1f
+I
+7d
+I
+25
+I
+8b
+I
+2c
+I
+99
+I
+31
+I
+a7
+I
+43
+I
+b5
+I
+6a
+I
+cc
+I
+68
+I
+db
+I
+6a
+I
+dd
+I
+6c
+I
+eb
+I
+71
+I
+f9
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0
+x
+13
+attach_method
+p
+7
+I
+0
+I
+1
+I
+9
+I
+2
+I
+12
+I
+4
+I
+35
+x
+55
+/Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
+p
+0