coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/html.rb

@@ -2,22 +2,42 @@ module CodeRay
 module Scanners
 
   # HTML Scanner
+  # 
+  # Alias: +xhtml+
+  # 
+  # See also: Scanners::XML
   class HTML < Scanner
 
-    include Streamable
     register_for :html
     
     KINDS_NOT_LOC = [
       :comment, :doctype, :preprocessor,
       :tag, :attribute_name, :operator,
-      :attribute_value, :delimiter, :content,
-      :plain, :entity, :error
-    ]
-
-    ATTR_NAME = /[\w.:-]+/
-    ATTR_VALUE_UNQUOTED = ATTR_NAME
-    TAG_END = /\/?>/
-    HEX = /[0-9a-fA-F]/
+      :attribute_value, :string,
+      :plain, :entity, :error,
+    ]  # :nodoc:
+    
+    EVENT_ATTRIBUTES = %w(
+      onabort onafterprint onbeforeprint onbeforeunload onblur oncanplay
+      oncanplaythrough onchange onclick oncontextmenu oncuechange ondblclick
+      ondrag ondragdrop ondragend ondragenter ondragleave ondragover
+      ondragstart ondrop ondurationchange onemptied onended onerror onfocus
+      onformchange onforminput onhashchange oninput oninvalid onkeydown
+      onkeypress onkeyup onload onloadeddata onloadedmetadata onloadstart
+      onmessage onmousedown onmousemove onmouseout onmouseover onmouseup
+      onmousewheel onmove onoffline ononline onpagehide onpageshow onpause
+      onplay onplaying onpopstate onprogress onratechange onreadystatechange
+      onredo onreset onresize onscroll onseeked onseeking onselect onshow
+      onstalled onstorage onsubmit onsuspend ontimeupdate onundo onunload
+      onvolumechange onwaiting
+    )
+    
+    IN_ATTRIBUTE = WordList::CaseIgnoring.new(nil).
+      add(EVENT_ATTRIBUTES, :script)
+    
+    ATTR_NAME = /[\w.:-]+/  # :nodoc:
+    TAG_END = /\/?>/  # :nodoc:
+    HEX = /[0-9a-fA-F]/  # :nodoc:
     ENTITY = /
       &
       (?:
@@ -31,152 +51,203 @@ module Scanners
         )
       )
       ;
-    /ox
-
+    /ox  # :nodoc:
+    
     PLAIN_STRING_CONTENT = {
       "'" => /[^&'>\n]+/,
       '"' => /[^&">\n]+/,
-    }
-
+    }  # :nodoc:
+    
     def reset
       super
       @state = :initial
+      @plain_string_content = nil
     end
-
-  private
+    
+  protected
+    
     def setup
       @state = :initial
       @plain_string_content = nil
     end
-
-    def scan_tokens tokens, options
-
-      state = @state
+    
+    def scan_java_script encoder, code
+      if code && !code.empty?
+        @java_script_scanner ||= Scanners::JavaScript.new '', :keep_tokens => true
+        # encoder.begin_group :inline
+        @java_script_scanner.tokenize code, :tokens => encoder
+        # encoder.end_group :inline
+      end
+    end
+    
+    def scan_tokens encoder, options
+      state = options[:state] || @state
       plain_string_content = @plain_string_content
-
+      in_tag = in_attribute = nil
+      
+      encoder.begin_group :string if state == :attribute_value_string
+      
       until eos?
-
-        kind = nil
-        match = nil
-
-        if scan(/\s+/m)
-          kind = :space
-
+        
+        if state != :in_special_tag && match = scan(/\s+/m)
+          encoder.text_token match, :space
+          
         else
-
+          
           case state
-
+          
           when :initial
-            if scan(/<!--.*?-->/m)
-              kind = :comment
-            elsif scan(/<!DOCTYPE.*?>/m)
-              kind = :doctype
-            elsif scan(/<\?xml.*?\?>/m)
-              kind = :preprocessor
-            elsif scan(/<\?.*?\?>|<%.*?%>/m)
-              kind = :comment
-            elsif scan(/<\/[-\w.:]*>/m)
-              kind = :tag
-            elsif match = scan(/<[-\w.:]+>?/m)
-              kind = :tag
-              state = :attribute unless match[-1] == ?>
-            elsif scan(/[^<>&]+/)
-              kind = :plain
-            elsif scan(/#{ENTITY}/ox)
-              kind = :entity
-            elsif scan(/[<>&]/)
-              kind = :error
+            if match = scan(/<!--(?:.*?-->|.*)/m)
+              encoder.text_token match, :comment
+            elsif match = scan(/<!DOCTYPE(?:.*?>|.*)/m)
+              encoder.text_token match, :doctype
+            elsif match = scan(/<\?xml(?:.*?\?>|.*)/m)
+              encoder.text_token match, :preprocessor
+            elsif match = scan(/<\?(?:.*?\?>|.*)/m)
+              encoder.text_token match, :comment
+            elsif match = scan(/<\/[-\w.:]*>?/m)
+              in_tag = nil
+              encoder.text_token match, :tag
+            elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m)
+              encoder.text_token match, :tag
+              in_tag = self[1]
+              if self[2]
+                state = :in_special_tag if in_tag
+              else
+                state = :attribute
+              end
+            elsif match = scan(/[^<>&]+/)
+              encoder.text_token match, :plain
+            elsif match = scan(/#{ENTITY}/ox)
+              encoder.text_token match, :entity
+            elsif match = scan(/[<>&]/)
+              in_tag = nil
+              encoder.text_token match, :error
             else
-              raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
+              raise_inspect '[BUG] else-case reached with state %p' % [state], encoder
             end
-
+            
           when :attribute
-            if scan(/#{TAG_END}/o)
-              kind = :tag
-              state = :initial
-            elsif scan(/#{ATTR_NAME}/o)
-              kind = :attribute_name
+            if match = scan(/#{TAG_END}/o)
+              encoder.text_token match, :tag
+              in_attribute = nil
+              if in_tag
+                state = :in_special_tag
+              else
+                state = :initial
+              end
+            elsif match = scan(/#{ATTR_NAME}/o)
+              in_attribute = IN_ATTRIBUTE[match]
+              encoder.text_token match, :attribute_name
               state = :attribute_equal
             else
-              kind = :error
-              getch
+              in_tag = nil
+              encoder.text_token getch, :error
             end
-
+            
           when :attribute_equal
-            if scan(/=/)
-              kind = :operator
+            if match = scan(/=/)  #/
+              encoder.text_token match, :operator
               state = :attribute_value
-            elsif scan(/#{ATTR_NAME}/o)
-              kind = :attribute_name
-            elsif scan(/#{TAG_END}/o)
-              kind = :tag
-              state = :initial
-            elsif scan(/./)
-              kind = :error
+            elsif scan(/#{ATTR_NAME}/o) || scan(/#{TAG_END}/o)
+              state = :attribute
+              next
+            else
+              encoder.text_token getch, :error
               state = :attribute
             end
-
+            
           when :attribute_value
-            if scan(/#{ATTR_VALUE_UNQUOTED}/o)
-              kind = :attribute_value
+            if match = scan(/#{ATTR_NAME}/o)
+              encoder.text_token match, :attribute_value
               state = :attribute
             elsif match = scan(/["']/)
-              tokens << [:open, :string]
-              state = :attribute_value_string
-              plain_string_content = PLAIN_STRING_CONTENT[match]
-              kind = :delimiter
-            elsif scan(/#{TAG_END}/o)
-              kind = :tag
+              if in_attribute == :script
+                encoder.begin_group :inline
+                encoder.text_token match, :inline_delimiter
+                if scan(/javascript:[ \t]*/)
+                  encoder.text_token matched, :comment
+                end
+                code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/)
+                scan_java_script encoder, code
+                match = scan(/["']/)
+                encoder.text_token match, :inline_delimiter if match
+                encoder.end_group :inline
+                state = :attribute
+                in_attribute = nil
+              else
+                encoder.begin_group :string
+                state = :attribute_value_string
+                plain_string_content = PLAIN_STRING_CONTENT[match]
+                encoder.text_token match, :delimiter
+              end
+            elsif match = scan(/#{TAG_END}/o)
+              encoder.text_token match, :tag
               state = :initial
             else
-              kind = :error
-              getch
+              encoder.text_token getch, :error
             end
-
+            
           when :attribute_value_string
-            if scan(plain_string_content)
-              kind = :content
-            elsif scan(/['"]/)
-              tokens << [matched, :delimiter]
-              tokens << [:close, :string]
+            if match = scan(plain_string_content)
+              encoder.text_token match, :content
+            elsif match = scan(/['"]/)
+              encoder.text_token match, :delimiter
+              encoder.end_group :string
               state = :attribute
-              next
-            elsif scan(/#{ENTITY}/ox)
-              kind = :entity
-            elsif scan(/&/)
-              kind = :content
-            elsif scan(/[\n>]/)
-              tokens << [:close, :string]
-              kind = :error
+            elsif match = scan(/#{ENTITY}/ox)
+              encoder.text_token match, :entity
+            elsif match = scan(/&/)
+              encoder.text_token match, :content
+            elsif match = scan(/[\n>]/)
+              encoder.end_group :string
               state = :initial
+              encoder.text_token match, :error
             end
-
+            
+          when :in_special_tag
+            case in_tag
+            when 'script'
+              encoder.text_token match, :space if match = scan(/[ \t]*\n/)
+              if scan(/(\s*<!--)(?:(.*?)(-->)|(.*))/m)
+                code = self[2] || self[4]
+                closing = self[3]
+                encoder.text_token self[1], :comment
+              else
+                code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/)
+                closing = false
+              end
+              unless code.empty?
+                encoder.begin_group :inline
+                scan_java_script encoder, code
+                encoder.end_group :inline
+              end
+              encoder.text_token closing, :comment if closing
+              state = :initial
+            else
+              raise 'unknown special tag: %p' % [in_tag]
+            end
+            
           else
-            raise_inspect 'Unknown state: %p' % [state], tokens
-
+            raise_inspect 'Unknown state: %p' % [state], encoder
+            
           end
-
-        end
-
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, state
+          
         end
-        raise_inspect 'Empty token', tokens unless match
-
-        tokens << [match, kind]
+        
       end
-
+      
       if options[:keep_state]
         @state = state
         @plain_string_content = plain_string_content
       end
-
-      tokens
+      
+      encoder.end_group :string if state == :attribute_value_string
+      
+      encoder
     end
-
+    
   end
-
+  
 end
 end

data/lib/coderay/scanners/java.rb

@@ -1,11 +1,12 @@
 module CodeRay
 module Scanners
-
+  
+  # Scanner for Java.
   class Java < Scanner
-
-    include Streamable
+    
     register_for :java
-    helper :builtin_types
+    
+    autoload :BuiltinTypes, 'coderay/scanners/java/builtin_types'
     
     # http://java.sun.com/docs/books/tutorial/java/nutsandbolts/_keywords.html
     KEYWORDS = %w[
@@ -13,63 +14,64 @@ module Scanners
       finally for if instanceof import new package
       return switch throw try typeof while
       debugger export
-    ]
-    RESERVED = %w[ const goto ]
-    CONSTANTS = %w[ false null true ]
-    MAGIC_VARIABLES = %w[ this super ]
+    ]  # :nodoc:
+    RESERVED = %w[ const goto ]  # :nodoc:
+    CONSTANTS = %w[ false null true ]  # :nodoc:
+    MAGIC_VARIABLES = %w[ this super ]  # :nodoc:
     TYPES = %w[
       boolean byte char class double enum float int interface long
       short void
-    ] << '[]'  # because int[] should be highlighted as a type
+    ] << '[]'  # :nodoc: because int[] should be highlighted as a type
     DIRECTIVES = %w[
       abstract extends final implements native private protected public
       static strictfp synchronized throws transient volatile
-    ]
+    ]  # :nodoc:
     
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(RESERVED, :reserved).
-      add(CONSTANTS, :pre_constant).
+      add(CONSTANTS, :predefined_constant).
       add(MAGIC_VARIABLES, :local_variable).
       add(TYPES, :type).
-      add(BuiltinTypes::List, :pre_type).
+      add(BuiltinTypes::List, :predefined_type).
       add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
-      add(DIRECTIVES, :directive)
+      add(DIRECTIVES, :directive)  # :nodoc:
 
-    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
     STRING_CONTENT_PATTERN = {
       "'" => /[^\\']+/,
       '"' => /[^\\"]+/,
       '/' => /[^\\\/]+/,
-    }
-    IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
+    }  # :nodoc:
+    IDENT = /[a-zA-Z_][A-Za-z_0-9]*/  # :nodoc:
+    
+  protected
     
-    def scan_tokens tokens, options
+    def scan_tokens encoder, options
 
       state = :initial
       string_delimiter = nil
-      import_clause = class_name_follows = last_token_dot = false
+      package_name_expected = false
+      class_name_follows = false
+      last_token_dot = false
 
       until eos?
 
-        kind = nil
-        match = nil
-        
         case state
 
         when :initial
 
           if match = scan(/ \s+ | \\\n /x)
-            tokens << [match, :space]
+            encoder.text_token match, :space
             next
           
           elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
-            tokens << [match, :comment]
+            encoder.text_token match, :comment
             next
           
-          elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
-            kind = :include
+          elsif package_name_expected && match = scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
+            encoder.text_token match, package_name_expected
           
           elsif match = scan(/ #{IDENT} | \[\] /ox)
             kind = IDENT_KIND[match]
@@ -79,95 +81,91 @@ module Scanners
               kind = :class
               class_name_follows = false
             else
-              import_clause = true if match == 'import'
-              class_name_follows = true if match == 'class' || match == 'interface'
+              case match
+              when 'import'
+                package_name_expected = :include
+              when 'package'
+                package_name_expected = :namespace
+              when 'class', 'interface'
+                class_name_follows = true
+              end
             end
+            encoder.text_token match, kind
           
-          elsif scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
-            kind = :operator
+          elsif match = scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
+            encoder.text_token match, :operator
           
-          elsif scan(/;/)
-            import_clause = false
-            kind = :operator
+          elsif match = scan(/;/)
+            package_name_expected = false
+            encoder.text_token match, :operator
           
-          elsif scan(/\{/)
+          elsif match = scan(/\{/)
             class_name_follows = false
-            kind = :operator
+            encoder.text_token match, :operator
           
           elsif check(/[\d.]/)
-            if scan(/0[xX][0-9A-Fa-f]+/)
-              kind = :hex
-            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
-              kind = :oct
-            elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
-              kind = :float
-            elsif scan(/\d+[lL]?/)
-              kind = :integer
+            if match = scan(/0[xX][0-9A-Fa-f]+/)
+              encoder.text_token match, :hex
+            elsif match = scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              encoder.text_token match, :octal
+            elsif match = scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+              encoder.text_token match, :float
+            elsif match = scan(/\d+[lL]?/)
+              encoder.text_token match, :integer
             end
 
           elsif match = scan(/["']/)
-            tokens << [:open, :string]
             state = :string
+            encoder.begin_group state
             string_delimiter = match
-            kind = :delimiter
+            encoder.text_token match, :delimiter
 
-          elsif scan(/ @ #{IDENT} /ox)
-            kind = :annotation
+          elsif match = scan(/ @ #{IDENT} /ox)
+            encoder.text_token match, :annotation
 
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
 
           end
 
         when :string
-          if scan(STRING_CONTENT_PATTERN[string_delimiter])
-            kind = :content
+          if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+            encoder.text_token match, :content
           elsif match = scan(/["'\/]/)
-            tokens << [match, :delimiter]
-            tokens << [:close, state]
-            string_delimiter = nil
+            encoder.text_token match, :delimiter
+            encoder.end_group state
             state = :initial
-            next
+            string_delimiter = nil
           elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
             if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
-              kind = :content
+              encoder.text_token match, :content
             else
-              kind = :char
+              encoder.text_token match, :char
             end
-          elsif scan(/\\./m)
-            kind = :content
-          elsif scan(/ \\ | $ /x)
-            tokens << [:close, state]
-            kind = :error
+          elsif match = scan(/\\./m)
+            encoder.text_token match, :content
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group state
             state = :initial
+            encoder.text_token match, :error
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
 
         else
-          raise_inspect 'Unknown state', tokens
+          raise_inspect 'Unknown state', encoder
 
         end
-
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
-        end
-        raise_inspect 'Empty token', tokens unless match
         
         last_token_dot = match == '.'
         
-        tokens << [match, kind]
-
       end
 
       if state == :string
-        tokens << [:close, state]
+        encoder.end_group state
       end
 
-      tokens
+      encoder
     end
 
   end