coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/java/builtin_types.rb

@@ -3,6 +3,7 @@ module Scanners
   
   module Java::BuiltinTypes  # :nodoc:
     
+    #:nocov:
     List = %w[
       AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
       AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
@@ -412,6 +413,7 @@ module Scanners
       XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
       XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
     ]
+    #:nocov:
     
   end
   

data/lib/coderay/scanners/java_script.rb

@@ -1,28 +1,29 @@
 module CodeRay
 module Scanners
-
+  
+  # Scanner for JavaScript.
+  # 
+  # Aliases: +ecmascript+, +ecma_script+, +javascript+
   class JavaScript < Scanner
-
-    include Streamable
-
+    
     register_for :java_script
     file_extension 'js'
-
+    
     # The actual JavaScript keywords.
     KEYWORDS = %w[
       break case catch continue default delete do else
       finally for function if in instanceof new
       return switch throw try typeof var void while with
-    ]
+    ]  # :nodoc:
     PREDEFINED_CONSTANTS = %w[
-      false null true undefined
-    ]
+      false null true undefined NaN Infinity
+    ]  # :nodoc:
     
-    MAGIC_VARIABLES = %w[ this arguments ]  # arguments was introduced in JavaScript 1.4
+    MAGIC_VARIABLES = %w[ this arguments ]  # :nodoc: arguments was introduced in JavaScript 1.4
     
     KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
       case delete in instanceof new return throw typeof with
-    ]
+    ]  # :nodoc:
     
     # Reserved for future use.
     RESERVED_WORDS = %w[
@@ -30,68 +31,66 @@ module Scanners
       final float goto implements import int interface long native package
       private protected public short static super synchronized throws transient
       volatile
-    ]
+    ]  # :nodoc:
     
     IDENT_KIND = WordList.new(:ident).
       add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
       add(MAGIC_VARIABLES, :local_variable).
-      add(KEYWORDS, :keyword)
-
-    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
-    REGEXP_ESCAPE =  / [bBdDsSwW] /x
+      add(KEYWORDS, :keyword)  # :nodoc:
+    
+    ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x  # :nodoc:
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x  # :nodoc:
+    REGEXP_ESCAPE =  / [bBdDsSwW] /x  # :nodoc:
     STRING_CONTENT_PATTERN = {
       "'" => /[^\\']+/,
       '"' => /[^\\"]+/,
       '/' => /[^\\\/]+/,
-    }
+    }  # :nodoc:
     KEY_CHECK_PATTERN = {
       "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx,
       '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx,
-    }
-
-    def scan_tokens tokens, options
-
+    }  # :nodoc:
+    
+  protected
+    
+    def scan_tokens encoder, options
+      
       state = :initial
       string_delimiter = nil
       value_expected = true
       key_expected = false
       function_expected = false
-
+      
       until eos?
-
-        kind = nil
-        match = nil
         
         case state
-
+          
         when :initial
-
+          
           if match = scan(/ \s+ | \\\n /x)
             value_expected = true if !value_expected && match.index(?\n)
-            tokens << [match, :space]
-            next
-
-          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            encoder.text_token match, :space
+            
+          elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
             value_expected = true
-            kind = :comment
-
+            encoder.text_token match, :comment
+            
           elsif check(/\.?\d/)
             key_expected = value_expected = false
-            if scan(/0[xX][0-9A-Fa-f]+/)
-              kind = :hex
-            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
-              kind = :oct
-            elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
-              kind = :float
-            elsif scan(/\d+/)
-              kind = :integer
+            if match = scan(/0[xX][0-9A-Fa-f]+/)
+              encoder.text_token match, :hex
+            elsif match = scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              encoder.text_token match, :octal
+            elsif match = scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+              encoder.text_token match, :float
+            elsif match = scan(/\d+/)
+              encoder.text_token match, :integer
             end
-          
+            
           elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
-            # FIXME: scan over nested tags
-            xml_scanner.tokenize match
+            # TODO: scan over nested tags
+            xml_scanner.tokenize match, :tokens => encoder
             value_expected = false
             next
             
@@ -100,12 +99,12 @@ module Scanners
             last_operator = match[-1]
             key_expected = (last_operator == ?{) || (last_operator == ?,)
             function_expected = false
-            kind = :operator
-
-          elsif scan(/ [)\]}]+ /x)
+            encoder.text_token match, :operator
+            
+          elsif match = scan(/ [)\]}]+ /x)
             function_expected = key_expected = value_expected = false
-            kind = :operator
-
+            encoder.text_token match, :operator
+            
           elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
             kind = IDENT_KIND[match]
             value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
@@ -123,101 +122,91 @@ module Scanners
             end
             function_expected = (kind == :keyword) && (match == 'function')
             key_expected = false
-          
+            encoder.text_token match, kind
+            
           elsif match = scan(/["']/)
             if key_expected && check(KEY_CHECK_PATTERN[match])
               state = :key
             else
               state = :string
             end
-            tokens << [:open, state]
+            encoder.begin_group state
             string_delimiter = match
-            kind = :delimiter
-
-          elsif value_expected && (match = scan(/\/(?=\S)/))
-            tokens << [:open, :regexp]
+            encoder.text_token match, :delimiter
+            
+          elsif value_expected && (match = scan(/\//))
+            encoder.begin_group :regexp
             state = :regexp
             string_delimiter = '/'
-            kind = :delimiter
-
-          elsif scan(/ \/ /x)
+            encoder.text_token match, :delimiter
+            
+          elsif match = scan(/ \/ /x)
             value_expected = true
             key_expected = false
-            kind = :operator
-
+            encoder.text_token match, :operator
+            
           else
-            getch
-            kind = :error
-
+            encoder.text_token getch, :error
+            
           end
-
+          
         when :string, :regexp, :key
-          if scan(STRING_CONTENT_PATTERN[string_delimiter])
-            kind = :content
+          if match = scan(STRING_CONTENT_PATTERN[string_delimiter])
+            encoder.text_token match, :content
           elsif match = scan(/["'\/]/)
-            tokens << [match, :delimiter]
+            encoder.text_token match, :delimiter
             if state == :regexp
               modifiers = scan(/[gim]+/)
-              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+              encoder.text_token modifiers, :modifier if modifiers && !modifiers.empty?
             end
-            tokens << [:close, state]
+            encoder.end_group state
             string_delimiter = nil
             key_expected = value_expected = false
             state = :initial
-            next
           elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
             if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
-              kind = :content
+              encoder.text_token match, :content
             else
-              kind = :char
+              encoder.text_token match, :char
             end
-          elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            kind = :char
-          elsif scan(/\\./m)
-            kind = :content
-          elsif scan(/ \\ | $ /x)
-            tokens << [:close, state]
-            kind = :error
+          elsif state == :regexp && match = scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/\\./m)
+            encoder.text_token match, :content
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group state
+            encoder.text_token match, :error
             key_expected = value_expected = false
             state = :initial
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
-
+          
         else
-          raise_inspect 'Unknown state', tokens
-
-        end
-
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
+          raise_inspect 'Unknown state', encoder
+          
         end
-        raise_inspect 'Empty token', tokens unless match
         
-        tokens << [match, kind]
-
       end
-
+      
       if [:string, :regexp].include? state
-        tokens << [:close, state]
+        encoder.end_group state
       end
-
-      tokens
+      
+      encoder
     end
-
+    
   protected
-
+    
     def reset_instance
       super
       @xml_scanner.reset if defined? @xml_scanner
     end
-
+    
     def xml_scanner
       @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
     end
-
+    
   end
   
 end

data/lib/coderay/scanners/json.rb

@@ -1,22 +1,24 @@
 module CodeRay
 module Scanners
   
+  # Scanner for JSON (JavaScript Object Notation).
   class JSON < Scanner
     
-    include Streamable
-    
     register_for :json
     file_extension 'json'
     
     KINDS_NOT_LOC = [
       :float, :char, :content, :delimiter,
       :error, :integer, :operator, :value,
-    ]
+    ]  # :nodoc:
+    
+    ESCAPE = / [bfnrt\\"\/] /x  # :nodoc:
+    UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x  # :nodoc:
     
-    ESCAPE = / [bfnrt\\"\/] /x
-    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x
+  protected
     
-    def scan_tokens tokens, options
+    # See http://json.org/ for a definition of the JSON lexic/grammar.
+    def scan_tokens encoder, options
       
       state = :initial
       stack = []
@@ -24,82 +26,67 @@ module Scanners
       
       until eos?
         
-        kind = nil
-        match = nil
-        
         case state
         
         when :initial
-          if match = scan(/ \s+ | \\\n /x)
-            tokens << [match, :space]
-            next
+          if match = scan(/ \s+ /x)
+            encoder.text_token match, :space
+          elsif match = scan(/"/)
+            state = key_expected ? :key : :string
+            encoder.begin_group state
+            encoder.text_token match, :delimiter
           elsif match = scan(/ [:,\[{\]}] /x)
-            kind = :operator
+            encoder.text_token match, :operator
             case match
-            when '{' then stack << :object; key_expected = true
-            when '[' then stack << :array
             when ':' then key_expected = false
             when ',' then key_expected = true if stack.last == :object
+            when '{' then stack << :object; key_expected = true
+            when '[' then stack << :array
             when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
             end
           elsif match = scan(/ true | false | null /x)
-            kind = :value
-          elsif match = scan(/-?(?:0|[1-9]\d*)/)
-            kind = :integer
-            if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
+            encoder.text_token match, :value
+          elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x)
+            if scan(/ \.\d+ (?:[eE][-+]?\d+)? | [eE][-+]? \d+ /x)
               match << matched
-              kind = :float
+              encoder.text_token match, :float
+            else
+              encoder.text_token match, :integer
             end
-          elsif match = scan(/"/)
-            state = key_expected ? :key : :string
-            tokens << [:open, state]
-            kind = :delimiter
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
           end
           
         when :string, :key
-          if scan(/[^\\"]+/)
-            kind = :content
-          elsif scan(/"/)
-            tokens << ['"', :delimiter]
-            tokens << [:close, state]
+          if match = scan(/[^\\"]+/)
+            encoder.text_token match, :content
+          elsif match = scan(/"/)
+            encoder.text_token match, :delimiter
+            encoder.end_group state
             state = :initial
-            next
-          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
-            kind = :char
-          elsif scan(/\\./m)
-            kind = :content
-          elsif scan(/ \\ | $ /x)
-            tokens << [:close, state]
-            kind = :error
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            encoder.text_token match, :char
+          elsif match = scan(/\\./m)
+            encoder.text_token match, :content
+          elsif match = scan(/ \\ | $ /x)
+            encoder.end_group state
+            encoder.text_token match, :error
             state = :initial
           else
-            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+            raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
           end
           
         else
-          raise_inspect 'Unknown state', tokens
+          raise_inspect 'Unknown state: %p' % [state], encoder
           
         end
-        
-        match ||= matched
-        if $CODERAY_DEBUG and not kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens
-        end
-        raise_inspect 'Empty token', tokens unless match
-        
-        tokens << [match, kind]
-        
       end
       
       if [:string, :key].include? state
-        tokens << [:close, state]
+        encoder.end_group state
       end
       
-      tokens
+      encoder
     end
     
   end