coderay 0.9.4 → 0.9.5

data/lib/README

@@ -18,7 +18,7 @@ And with line numbers.
 * is what everybody should have on their website
 * solves all your problems and makes the girls run after you
 
-Version: 0.9.4
+Version: 0.9.5
 Author:: murphy (Kornelius Kalnbach)
 Contact:: murphy rubychan de
 Website:: coderay.rubychan.de[http://coderay.rubychan.de]

data/lib/coderay.rb

@@ -134,7 +134,7 @@ module CodeRay
   # Minor: feature milestone
   # Teeny: development state, 0 for pre-release
   # Revision: Subversion Revision number (generated on rake gem:make)
-  VERSION = '0.9.4'
+  VERSION = '0.9.5'
 
   require 'coderay/tokens'
   require 'coderay/token_classes'

data/lib/coderay/encoders/html.rb

@@ -276,9 +276,13 @@ module Encoders
       when :begin_line
         @opened[0] = type
         if style = @css_style[@opened]
-          @out << style.sub('<span', '<div')
+          if style['class="']
+            @out << style.sub('class="', 'class="line ')
+          else
+            @out << style.sub('>', ' class="line">')
+          end
         else
-          @out << '<div>'
+          @out << '<span class="line">'
         end
         @opened << type
       when :end_line
@@ -289,7 +293,7 @@ module Encoders
             raise 'Malformed token stream: Trying to close a line (%p) \
               that is not open. Open are: %p.' % [type, @opened[1..-1]]
           end
-          @out << '</div>'
+          @out << '</span>'
           @opened.pop
         end
       

data/lib/coderay/encoders/html/numerization.rb

@@ -80,8 +80,8 @@ module Encoders
           line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
 
           line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
-          gsub!(/<\/div>\n/) { '</div>' }
-          gsub!(/\n/) { "<tt>\n</tt>" }
+          gsub!("</div>\n", '</div>')
+          gsub!("\n", "<tt>\n</tt>")
           wrap_in! line_numbers_table_tpl
           @wrapped_in = :div
 

data/lib/coderay/scanner.rb

@@ -68,15 +68,15 @@ module CodeRay
 
         def normify code
           code = code.to_s
-          if code.respond_to? :force_encoding
-            debug, $DEBUG = $DEBUG, false
-            begin
-              code.force_encoding 'utf-8'
-              code[/\z/]  # raises an ArgumentError when code contains a non-UTF-8 char
-            rescue ArgumentError
-              code.force_encoding 'binary'
-            ensure
-              $DEBUG = debug
+          if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
+            original_encoding = code.encoding
+            code.force_encoding 'Windows-1252'
+            unless code.valid_encoding?
+              code.force_encoding original_encoding
+              if code.encoding.name == 'UTF-8'
+                code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
+              end
+              code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
             end
           end
           code.to_unix
@@ -147,7 +147,12 @@ module CodeRay
 
       def string= code
         code = Scanner.normify(code)
-        super code
+        if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
+          reset_state
+          @string = code
+        else
+          super code
+        end
         reset_instance
       end
 

data/lib/coderay/scanners/css.rb

@@ -14,12 +14,11 @@ module Scanners
     ]
     
     module RE
-      NonASCII = /[\x80-\xFF]/
       Hex = /[0-9a-fA-F]/
       Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
       Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
-      NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
-      NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
+      NMChar = /[-_a-zA-Z0-9]|#{Escape}/
+      NMStart = /[_a-zA-Z]|#{Escape}/
       NL = /\r\n|\r|\n|\f/
       String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/  # FIXME: buggy regexp
       String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/  # FIXME: buggy regexp

data/lib/coderay/scanners/diff.rb

@@ -72,7 +72,7 @@ module Scanners
           elsif scan(/ .*/)
             kind = :comment
           elsif scan(/.+/)
-            tokens << [:begin_line, line_kind = :head]
+            tokens << [:begin_line, line_kind = :comment]
             kind = :plain
           else
             raise_inspect 'else case rached'

data/lib/coderay/scanners/php.rb

@@ -225,6 +225,12 @@ module Scanners
     end
     
     def scan_tokens tokens, options
+      if string.respond_to?(:encoding)
+        unless string.encoding == Encoding::ASCII_8BIT
+          self.string = string.encode Encoding::ASCII_8BIT,
+            :invalid => :replace, :undef => :replace, :replace => '?'
+        end
+      end
       
       if check(RE::PHP_START) ||  # starts with <?
        (match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?

data/lib/coderay/scanners/ruby.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 module CodeRay
 module Scanners
 
@@ -28,6 +29,16 @@ module Scanners
 
   private
     def scan_tokens tokens, options
+      if string.respond_to?(:encoding)
+        unless string.encoding == Encoding::UTF_8
+          self.string = string.encode Encoding::UTF_8,
+            :invalid => :replace, :undef => :replace, :replace => '?'
+        end
+        unicode = false
+      else
+        unicode = exist?(/[^\x00-\x7f]/)
+      end
+      
       last_token_dot = false
       value_expected = true
       heredocs = nil
@@ -35,7 +46,7 @@ module Scanners
       state = :initial
       depth = nil
       inline_block_stack = []
-      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+      
       
       patterns = Patterns  # avoid constant lookup
       
@@ -171,8 +182,13 @@ module Scanners
                 kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
               else
                 kind = patterns::IDENT_KIND[match]
-                if kind == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
-                  kind = :constant
+                if kind == :ident
+                  if match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
+                    kind = :constant
+                  elsif scan(/:(?= )/)
+                    match << ':'
+                    kind = :symbol
+                  end
                 elsif kind == :reserved
                   state = patterns::DEF_NEW_STATE[match]
                   value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
@@ -182,7 +198,8 @@ module Scanners
             
             elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
               kind = :ident
-              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+              value_expected = :set if check(unicode ? /#{patterns::VALUE_FOLLOWS}/uo :
+                                                       /#{patterns::VALUE_FOLLOWS}/o)
 
             # OPERATORS #
             elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
@@ -212,7 +229,8 @@ module Scanners
               kind = :delimiter
               state = patterns::StringState.new :string, match == '"', match  # important for streaming
 
-            elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
+            elsif match = scan(unicode ? /#{patterns::INSTANCE_VARIABLE}/uo :
+                                         /#{patterns::INSTANCE_VARIABLE}/o)
               kind = :instance_variable
 
             elsif value_expected and match = scan(/\//)
@@ -225,7 +243,8 @@ module Scanners
             elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
               kind = self[1] ? :float : :integer
 
-            elsif match = scan(/#{patterns::SYMBOL}/o)
+            elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
+                                         /#{patterns::SYMBOL}/o)
               case delim = match[1]
               when ?', ?"
                 tokens << [:open, :symbol]
@@ -237,11 +256,12 @@ module Scanners
                 kind = :symbol
               end
 
-            elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
+            elsif match = scan(/ -[>=]? | [+!~^]=? | [*|&]{1,2}=? | >>? /x)
               value_expected = :set
               kind = :operator
 
-            elsif value_expected and match = scan(/#{patterns::HEREDOC_OPEN}/o)
+            elsif value_expected and match = scan(unicode ? /#{patterns::HEREDOC_OPEN}/uo :
+                                                            /#{patterns::HEREDOC_OPEN}/o)
               indented = self[1] == '-'
               quote = self[3]
               delim = self[quote ? 4 : 2]
@@ -261,7 +281,8 @@ module Scanners
               state = patterns::StringState.new kind, interpreted, self[2]
               kind = :delimiter
 
-            elsif value_expected and match = scan(/#{patterns::CHARACTER}/o)
+            elsif value_expected and match = scan(unicode ? /#{patterns::CHARACTER}/uo :
+                                                            /#{patterns::CHARACTER}/o)
               kind = :integer
 
             elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
@@ -277,14 +298,16 @@ module Scanners
                 state = patterns::StringState.new :shell, true, match
               end
 
-            elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
+            elsif match = scan(unicode ? /#{patterns::GLOBAL_VARIABLE}/uo :
+                                         /#{patterns::GLOBAL_VARIABLE}/o)
               kind = :global_variable
 
-            elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
+            elsif match = scan(unicode ? /#{patterns::CLASS_VARIABLE}/uo :
+                                         /#{patterns::CLASS_VARIABLE}/o)
               kind = :class_variable
 
             else
-              if !unicode
+              if !unicode && !string.respond_to?(:encoding)
                 # check for unicode
                 debug, $DEBUG = $DEBUG, false
                 begin
@@ -300,7 +323,7 @@ module Scanners
                 next if unicode
               end
               kind = :error
-              match = getch
+              match = scan(unicode ? /./mu : /./m)
 
             end
 
@@ -322,7 +345,8 @@ module Scanners
               kind = :operator
             else
               state = :initial
-              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
+              if match = scan(unicode ? /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/uo :
+                                        /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/o)
                 kind = :class
               else
                 next
@@ -331,9 +355,11 @@ module Scanners
 
           elsif state == :undef_expected
             state = :undef_comma_expected
-            if match = scan(/#{patterns::METHOD_NAME_EX}/o)
+            if match = scan(unicode ? /#{patterns::METHOD_NAME_EX}/uo :
+                                      /#{patterns::METHOD_NAME_EX}/o)
               kind = :method
-            elsif match = scan(/#{patterns::SYMBOL}/o)
+            elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
+                                         /#{patterns::SYMBOL}/o)
               case delim = match[1]
               when ?', ?"
                 tokens << [:open, :symbol]

data/lib/coderay/scanners/ruby/patterns.rb

@@ -32,7 +32,18 @@ module Scanners
       add(RESERVED_WORDS, :reserved).
       add(PREDEFINED_CONSTANTS, :pre_constant)
 
-    IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
+    if /\w/u === '∑'
+      # MRI 1.8.6, 1.8.7
+      IDENT = /[^\W\d]\w*/
+    else
+      if //.respond_to? :encoding
+        # MRI 1.9.1, 1.9.2
+        IDENT = Regexp.new '[\p{L}\p{M}\p{Pc}\p{Sm}&&[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f]][\p{L}\p{M}\p{N}\p{Pc}\p{Sm}&&[^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]]*'
+      else
+        # JRuby, Rubinius
+        IDENT = /[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f][^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]*/
+      end
+    end
 
     METHOD_NAME = / #{IDENT} [?!]? /ox
     METHOD_NAME_OPERATOR = /
@@ -109,10 +120,12 @@ module Scanners
 
     # NOTE: This is not completely correct, but
     # nobody needs heredoc delimiters ending with \n.
+    # Also, delimiters starting with numbers are allowed.
+    # but they are more often than not a false positive.
     HEREDOC_OPEN = /
       << (-)?              # $1 = float
       (?:
-        ( [A-Za-z_0-9]+ )  # $2 = delim
+        ( #{IDENT} )       # $2 = delim
       |
         ( ["'`\/] )        # $3 = quote, type
         ( [^\n]*? ) \3     # $4 = delim

data/lib/coderay/scanners/sql.rb

@@ -56,7 +56,7 @@ module CodeRay module Scanners
           if scan(/ \s+ | \\\n /x)
             kind = :space
           
-          elsif scan(/^(?:--\s?|#).*/)
+          elsif scan(/(?:--\s?|#).*/)
             kind = :comment
             
           elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)

data/lib/coderay/styles/cycnus.rb

@@ -33,6 +33,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top }
 }
 .CodeRay .line_numbers tt { font-weight: bold }
 .CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .line { display: block; float: left; width: 100%; }
 .CodeRay .no { padding: 0px 4px }
 .CodeRay .code { width: 100% }
 

data/lib/coderay/styles/murphy.rb

@@ -32,6 +32,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
   text-align: right;
 }
 .CodeRay .line_numbers tt { font-weight: bold; }
+.CodeRay .line_numbers .highlighted { color: red }
+.CodeRay .line { display: block; float: left; width: 100%; }
 .CodeRay .no { padding: 0px 4px; }
 .CodeRay .code { width: 100%; }
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: coderay
 version: !ruby/object:Gem::Version 
-  hash: 51
+  hash: 49
   prerelease: false
   segments: 
   - 0
   - 9
-  - 4
-  version: 0.9.4
+  - 5
+  version: 0.9.5
 platform: ruby
 authors: 
 - murphy
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-01 00:00:00 +02:00
+date: 2010-09-28 00:00:00 +02:00
 default_executable: 
 dependencies: []