coderay 1.0.0.738.pre → 1.0.0.778.pre

data/lib/coderay/scanners/_map.rb

@@ -12,11 +12,12 @@ module Scanners
     :nitro => :nitro_xhtml,
     :pascal => :delphi,
     :patch => :diff,
-    :plain => :plaintext,
+    :plain => :text,
+    :plaintext => :text,
     :xhtml => :html,
     :yml => :yaml
   
-  default :plain
+  default :text
   
 end
 end

data/lib/coderay/scanners/c.rb

@@ -7,7 +7,7 @@ module Scanners
     register_for :c
     file_extension 'c'
     
-    RESERVED_WORDS = [
+    KEYWORDS = [
       'asm', 'break', 'case', 'continue', 'default', 'do',
       'else', 'enum', 'for', 'goto', 'if', 'return',
       'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
@@ -31,10 +31,10 @@ module Scanners
     ]  # :nodoc:
 
     IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_TYPES, :pre_type).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :pre_constant)  # :nodoc:
+      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
@@ -80,7 +80,7 @@ module Scanners
               match << matched
             else
               label_expected = false
-              if kind == :reserved
+              if kind == :keyword
                 case match
                 when 'case', 'default'
                   case_expected = true

data/lib/coderay/scanners/clojure.rb

@@ -14,7 +14,7 @@ module CodeRay
       ]  # :nodoc:
       
       CORE_FORMS = %w[
-        + - -> ->> .. / * < <= = == > >= accessor aclone add-classpath add-watch
+        + - -> ->> .. / * <= < = == >= > accessor aclone add-classpath add-watch
         agent agent-error agent-errors aget alength alias all-ns alter alter-meta!
         alter-var-root amap ancestors and apply areduce array-map aset aset-boolean
         aset-byte aset-char aset-double aset-float aset-int aset-long aset-short
@@ -74,20 +74,26 @@ module CodeRay
         with-in-str with-local-vars with-meta with-open with-out-str
         with-precision xml-seq zero? zipmap 
       ]  # :nodoc:
+      
       PREDEFINED_CONSTANTS = %w[
-        true false *1 *2 *3 *agent* *clojure-version* *command-line-args*
+        true false nil *1 *2 *3 *agent* *clojure-version* *command-line-args*
         *compile-files* *compile-path* *e *err* *file* *flush-on-newline*
         *in* *ns* *out* *print-dup* *print-length* *print-level* *print-meta*
         *print-readably* *read-eval* *warn-on-reflection*
       ]  # :nodoc:
       
-      IDENT_KIND = CaseIgnoringWordList.new(:ident).
+      IDENT_KIND = WordList.new(:ident).
         add(SPECIAL_FORMS, :reserved).
         add(CORE_FORMS, :reserved).
-        add(PREDEFINED_CONSTANTS, :pre_constant)
+        add(PREDEFINED_CONSTANTS, :predefined_constant)
+      
+      KEYWORD_NEXT_TOKEN_KIND = WordList.new(nil).
+        add(%w[ def defn defn- definline defmacro defmulti defmethod defstruct defonce declare ], :function).
+        add(%w[ ns ], :namespace).
+        add(%w[ defprotocol defrecord ], :class)
       
-      BASIC_IDENTIFIER = /[a-zA-Z$%*\/_+!?&<>\-=][a-zA-Z0-9$&*+!\/_?<>\-\#]*/
-      IDENTIFIER = /(?:[@']?(?:#{BASIC_IDENTIFIER}\.)*#{BASIC_IDENTIFIER}(?:\/#{BASIC_IDENTIFIER})?\.?)|\.\.?/
+      BASIC_IDENTIFIER = /[a-zA-Z$%*\/_+!?&<>\-=]=?[a-zA-Z0-9$&*+!\/_?<>\-\#]*/
+      IDENTIFIER = /(?!-\d)(?:(?:#{BASIC_IDENTIFIER}\.)*#{BASIC_IDENTIFIER}(?:\/#{BASIC_IDENTIFIER})?\.?)|\.\.?/
       SYMBOL = /::?#{IDENTIFIER}/o
       DIGIT = /\d/
       DIGIT10 = DIGIT
@@ -139,7 +145,7 @@ module CodeRay
       def scan_tokens encoder, options
         
         state = :initial
-        ident_kind = IDENT_KIND
+        kind = nil
         
         until eos?
           
@@ -147,16 +153,23 @@ module CodeRay
           when :initial
             if match = scan(/ \s+ | \\\n | , /x)
               encoder.text_token match, :space
-            elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|@/)
-              encoder.text_token match, :operator  # FIXME: was :operator_fat
+            elsif match = scan(/['`\(\[\)\]\{\}]|\#[({]|~@?|[@\^]/)
+              encoder.text_token match, :operator
             elsif match = scan(/;.*/)
-              encoder.text_token match, :comment
-            elsif match = scan(/\#\\(?:newline|space|.?)/)
+              encoder.text_token match, :comment  # FIXME: recognize (comment ...) too
+            elsif match = scan(/\#?\\(?:newline|space|.?)/)
               encoder.text_token match, :char
             elsif match = scan(/\#[ft]/)
-              encoder.text_token match, :pre_constant
+              encoder.text_token match, :predefined_constant
             elsif match = scan(/#{IDENTIFIER}/o)
-              encoder.text_token match, ident_kind[matched]
+              kind = IDENT_KIND[match]
+              encoder.text_token match, kind
+              if rest? && kind == :reserved
+                if kind = KEYWORD_NEXT_TOKEN_KIND[match]
+                  encoder.text_token match, :space if match = scan(/\s+/o)
+                  encoder.text_token match, kind if match = scan(/#{IDENTIFIER}/o)
+                end
+              end
             elsif match = scan(/#{SYMBOL}/o)
               encoder.text_token match, :symbol
             elsif match = scan(/\./)
@@ -168,7 +181,7 @@ module CodeRay
               encoder.begin_group state
               encoder.text_token match, :delimiter
             elsif match = scan(/#{NUM}/o) and not matched.empty?
-              encoder.text_token match, match[/[.e]/i] ? :float : :integer
+              encoder.text_token match, match[/[.e\/]/i] ? :float : :integer
             else
               encoder.text_token getch, :error
             end

data/lib/coderay/scanners/cpp.rb

@@ -11,7 +11,7 @@ module Scanners
     title 'C++'
     
     #-- http://www.cppreference.com/wiki/keywords/start
-    RESERVED_WORDS = [
+    KEYWORDS = [
       'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
       'case', 'catch', 'class', 'compl', 'const_cast',
       'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
@@ -40,11 +40,11 @@ module Scanners
     ]  # :nodoc:
     
     IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_TYPES, :pre_type).
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_TYPES, :predefined_type).
       add(PREDEFINED_VARIABLES, :local_variable).
       add(DIRECTIVES, :directive).
-      add(PREDEFINED_CONSTANTS, :pre_constant)  # :nodoc:
+      add(PREDEFINED_CONSTANTS, :predefined_constant)  # :nodoc:
 
     ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
@@ -94,7 +94,7 @@ module Scanners
               match << matched
             else
               label_expected = false
-              if kind == :reserved
+              if kind == :keyword
                 case match
                 when 'class'
                   state = :class_name_expected

data/lib/coderay/scanners/css.rb

@@ -9,7 +9,7 @@ module Scanners
       :comment,
       :class, :pseudo_class, :type,
       :constant, :directive,
-      :key, :value, :operator, :color, :float,
+      :key, :value, :operator, :color, :float, :string,
       :error, :important,
     ]  # :nodoc:
     

data/lib/coderay/scanners/html.rb

@@ -13,10 +13,28 @@ module Scanners
     KINDS_NOT_LOC = [
       :comment, :doctype, :preprocessor,
       :tag, :attribute_name, :operator,
-      :attribute_value, :delimiter, :content,
+      :attribute_value, :string,
       :plain, :entity, :error,
     ]  # :nodoc:
     
+    EVENT_ATTRIBUTES = %w(
+      onabort onafterprint onbeforeprint onbeforeunload onblur oncanplay
+      oncanplaythrough onchange onclick oncontextmenu oncuechange ondblclick
+      ondrag ondragdrop ondragend ondragenter ondragleave ondragover
+      ondragstart ondrop ondurationchange onemptied onended onerror onfocus
+      onformchange onforminput onhashchange oninput oninvalid onkeydown
+      onkeypress onkeyup onload onloadeddata onloadedmetadata onloadstart
+      onmessage onmousedown onmousemove onmouseout onmouseover onmouseup
+      onmousewheel onmove onoffline ononline onpagehide onpageshow onpause
+      onplay onplaying onpopstate onprogress onratechange onreadystatechange
+      onredo onreset onresize onscroll onseeked onseeking onselect onshow
+      onstalled onstorage onsubmit onsuspend ontimeupdate onundo onunload
+      onvolumechange onwaiting
+    )
+    
+    IN_ATTRIBUTE = CaseIgnoringWordList.new(nil).
+      add(EVENT_ATTRIBUTES, :script)
+    
     ATTR_NAME = /[\w.:-]+/  # :nodoc:
     TAG_END = /\/?>/  # :nodoc:
     HEX = /[0-9a-fA-F]/  # :nodoc:
@@ -70,7 +88,7 @@ module Scanners
       
       until eos?
         
-        if match = scan(/\s+/m)
+        if state != :in_special_tag && match = scan(/\s+/m)
           encoder.text_token match, :space
           
         else
@@ -78,24 +96,6 @@ module Scanners
           case state
           
           when :initial
-            case in_tag
-            when 'script'
-              if scan(/(\s*<!--)(?:(.*?)(-->)|(.*))/m)
-                code = self[2] || self[4]
-                closing = self[3]
-                encoder.text_token self[1], :comment
-              else
-                code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/)
-                closing = false
-              end
-              unless code.empty?
-                encoder.begin_group :inline
-                scan_java_script encoder, code
-                encoder.end_group :inline
-              end
-              encoder.text_token closing, :comment if closing
-            end
-            next if eos?
             if match = scan(/<!--(?:.*?-->|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<!DOCTYPE(?:.*?>|.*)/m)
@@ -105,17 +105,22 @@ module Scanners
             elsif match = scan(/<\?(?:.*?\?>|.*)|<%(?:.*?%>|.*)/m)
               encoder.text_token match, :comment
             elsif match = scan(/<\/[-\w.:]*>?/m)
-              encoder.text_token match, :tag
               in_tag = nil
+              encoder.text_token match, :tag
             elsif match = scan(/<(?:(script)|[-\w.:]+)(>)?/m)
               encoder.text_token match, :tag
               in_tag = self[1]
-              state = :attribute unless self[2]
+              if self[2]
+                state = :in_special_tag if in_tag
+              else
+                state = :attribute
+              end
             elsif match = scan(/[^<>&]+/)
               encoder.text_token match, :plain
             elsif match = scan(/#{ENTITY}/ox)
               encoder.text_token match, :entity
             elsif match = scan(/[<>&]/)
+              in_tag = nil
               encoder.text_token match, :error
             else
               raise_inspect '[BUG] else-case reached with state %p' % [state], encoder
@@ -125,19 +130,22 @@ module Scanners
             if match = scan(/#{TAG_END}/o)
               encoder.text_token match, :tag
               in_attribute = nil
-              state = :initial
-            elsif match = scan(/#{ATTR_NAME}/o)
-              if match.downcase == 'onclick'
-                in_attribute = 'script'
+              if in_tag
+                state = :in_special_tag
+              else
+                state = :initial
               end
+            elsif match = scan(/#{ATTR_NAME}/o)
+              in_attribute = IN_ATTRIBUTE[match]
               encoder.text_token match, :attribute_name
               state = :attribute_equal
             else
+              in_tag = nil
               encoder.text_token getch, :error
             end
             
           when :attribute_equal
-            if match = scan(/=/)
+            if match = scan(/=/)  #/
               encoder.text_token match, :operator
               state = :attribute_value
             elsif scan(/#{ATTR_NAME}/o) || scan(/#{TAG_END}/o)
@@ -153,10 +161,10 @@ module Scanners
               encoder.text_token match, :attribute_value
               state = :attribute
             elsif match = scan(/["']/)
-              if in_attribute == 'script'
+              if in_attribute == :script
                 encoder.begin_group :inline
                 encoder.text_token match, :inline_delimiter
-                if scan(/javascript:\s*/)
+                if scan(/javascript:[ \t]*/)
                   encoder.text_token matched, :comment
                 end
                 code = scan_until(match == '"' ? /(?="|\z)/ : /(?='|\z)/)
@@ -196,6 +204,29 @@ module Scanners
               encoder.text_token match, :error
             end
             
+          when :in_special_tag
+            case in_tag
+            when 'script'
+              encoder.text_token match, :space if match = scan(/[ \t]*\n/)
+              if scan(/(\s*<!--)(?:(.*?)(-->)|(.*))/m)
+                code = self[2] || self[4]
+                closing = self[3]
+                encoder.text_token self[1], :comment
+              else
+                code = scan_until(/(?=(?:\n\s*)?<\/script>)|\z/)
+                closing = false
+              end
+              unless code.empty?
+                encoder.begin_group :inline
+                scan_java_script encoder, code
+                encoder.end_group :inline
+              end
+              encoder.text_token closing, :comment if closing
+              state = :initial
+            else
+              raise 'unknown special tag: %p' % [in_tag]
+            end
+            
           else
             raise_inspect 'Unknown state: %p' % [state], encoder
             

data/lib/coderay/scanners/java.rb

@@ -30,10 +30,10 @@ module Scanners
     IDENT_KIND = WordList.new(:ident).
       add(KEYWORDS, :keyword).
       add(RESERVED, :reserved).
-      add(CONSTANTS, :pre_constant).
+      add(CONSTANTS, :predefined_constant).
       add(MAGIC_VARIABLES, :local_variable).
       add(TYPES, :type).
-      add(BuiltinTypes::List, :pre_type).
+      add(BuiltinTypes::List, :predefined_type).
       add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
       add(DIRECTIVES, :directive)  # :nodoc:
 

data/lib/coderay/scanners/java_script.rb

@@ -35,7 +35,7 @@ module Scanners
     
     IDENT_KIND = WordList.new(:ident).
       add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
       add(MAGIC_VARIABLES, :local_variable).
       add(KEYWORDS, :keyword)  # :nodoc:
     

data/lib/coderay/scanners/nitro_xhtml.rb

@@ -89,7 +89,7 @@ module Scanners
       
       until eos?
         
-        if (match = scan_until(/(?=#{START_OF_RUBY})/o) || match = scan_until(/\z/)) and not match.empty?
+        if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_rest) and not match.empty?
           @html_scanner.tokenize match
           
         elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)

data/lib/coderay/scanners/php.rb

@@ -183,12 +183,12 @@ module Scanners
       
       IDENT_KIND = CaseIgnoringWordList.new(:ident).
         add(KEYWORDS, :reserved).
-        add(TYPES, :pre_type).
+        add(TYPES, :predefined_type).
         add(LANGUAGE_CONSTRUCTS, :reserved).
         add(BUILTIN_FUNCTIONS, :predefined).
-        add(CLASSES, :pre_constant).
+        add(CLASSES, :predefined_constant).
         add(EXCEPTIONS, :exception).
-        add(CONSTANTS, :pre_constant)
+        add(CONSTANTS, :predefined_constant)
       
       VARIABLE_KIND = WordList.new(:local_variable).
         add(PREDEFINED, :predefined)
@@ -261,7 +261,7 @@ module Scanners
             label_expected = true
             states << :php
           else
-            match = scan_until(/(?=#{RE::PHP_START})/o) || scan_until(/\z/)
+            match = scan_until(/(?=#{RE::PHP_START})/o) || scan_rest
             @html_scanner.tokenize match unless match.empty?
           end
         

data/lib/coderay/scanners/python.rb

@@ -58,7 +58,7 @@ module Scanners
       add(KEYWORDS, :keyword).
       add(OLD_KEYWORDS, :old_keyword).
       add(PREDEFINED_METHODS_AND_TYPES, :predefined).
-      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
+      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :predefined_constant).
       add(PREDEFINED_EXCEPTIONS, :exception)  # :nodoc:
     
     NAME = / [^\W\d] \w* /x  # :nodoc:
@@ -107,7 +107,7 @@ module Scanners
       string_raw = false
       string_type = nil
       docstring_coming = match?(/#{DOCSTRING_COMING}/o)
-      import_clause = class_name_follows = last_token_dot = false
+      last_token_dot = false
       unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
       from_import_state = []
       

data/lib/coderay/scanners/rhtml.rb

@@ -44,7 +44,7 @@ module Scanners
       
       until eos?
         
-        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
+        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_rest) and not match.empty?
           @html_scanner.tokenize match, :tokens => encoder
           
         elsif match = scan(/#{ERB_RUBY_BLOCK}/o)

data/lib/coderay/scanners/ruby.rb

@@ -8,13 +8,6 @@ module Scanners
   #
   # It is optimized for HTML highlighting, and is not very useful for
   # parsing or pretty printing.
-  #
-  # For now, I think it's better than the scanners in VIM or Syntax, or
-  # any highlighter I was able to find, except Caleb's RubyLexer.
-  #
-  # I hope it's also better than the rdoc/irb lexer.
-  # 
-  # Alias: +irb+
   class Ruby < Scanner
     
     register_for :ruby
@@ -31,10 +24,8 @@ module Scanners
     
     def scan_tokens encoder, options
       
-      patterns = Patterns  # avoid constant lookup
-      
       state = @state
-      if state.instance_of? self.class::StringState
+      if state && state.instance_of?(self.class::StringState)
         encoder.begin_group state.type
       end
       
@@ -50,6 +41,8 @@ module Scanners
       # def_object_stack = nil
       # def_object_paren_depth = 0
       
+      patterns = Patterns  # avoid constant lookup
+      
       unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
       
       until eos?
@@ -339,7 +332,7 @@ module Scanners
           
         else  # StringState
           
-          match = scan_until(state.pattern) || scan_until(/\z/)
+          match = scan_until(state.pattern) || scan_rest
           unless match.empty?
             encoder.text_token match, :content
             break if eos?