coderay 0.8.357 → 0.9.1

data/lib/coderay/scanners/plaintext.rb

@@ -4,9 +4,12 @@ module Scanners
   class Plaintext < Scanner
 
     register_for :plaintext, :plain
+    title 'Plain text'
     
     include Streamable
-
+    
+    KINDS_NOT_LOC = [:plain]
+    
     def scan_tokens tokens, options
       text = (scan_until(/\z/) || '')
       tokens << [text, :plain]

data/lib/coderay/scanners/python.rb

@@ -0,0 +1,285 @@
+module CodeRay
+module Scanners
+  
+  # Bases on pygments' PythonLexer, see
+  # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
+  class Python < Scanner
+    
+    include Streamable
+    
+    register_for :python
+    file_extension 'py'
+    
+    KEYWORDS = [
+      'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
+      'del', 'elif', 'else', 'except', 'finally', 'for',
+      'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
+      'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
+      'nonlocal',  # new in Python 3
+    ]
+    
+    OLD_KEYWORDS = [
+      'exec', 'print',  # gone in Python 3
+    ]
+    
+    PREDEFINED_METHODS_AND_TYPES = %w[
+      __import__ abs all any apply basestring bin bool buffer
+      bytearray bytes callable chr classmethod cmp coerce compile
+      complex delattr dict dir divmod enumerate eval execfile exit
+      file filter float frozenset getattr globals hasattr hash hex id
+      input int intern isinstance issubclass iter len list locals
+      long map max min next object oct open ord pow property range
+      raw_input reduce reload repr reversed round set setattr slice
+      sorted staticmethod str sum super tuple type unichr unicode
+      vars xrange zip
+    ]
+    
+    PREDEFINED_EXCEPTIONS = %w[
+      ArithmeticError AssertionError AttributeError
+      BaseException DeprecationWarning EOFError EnvironmentError
+      Exception FloatingPointError FutureWarning GeneratorExit IOError
+      ImportError ImportWarning IndentationError IndexError KeyError
+      KeyboardInterrupt LookupError MemoryError NameError
+      NotImplemented NotImplementedError OSError OverflowError
+      OverflowWarning PendingDeprecationWarning ReferenceError
+      RuntimeError RuntimeWarning StandardError StopIteration
+      SyntaxError SyntaxWarning SystemError SystemExit TabError
+      TypeError UnboundLocalError UnicodeDecodeError
+      UnicodeEncodeError UnicodeError UnicodeTranslateError
+      UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
+    ]
+    
+    PREDEFINED_VARIABLES_AND_CONSTANTS = [
+      'False', 'True', 'None', # "keywords" since Python 3
+      'self', 'Ellipsis', 'NotImplemented',
+    ]
+    
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(OLD_KEYWORDS, :old_keyword).
+      add(PREDEFINED_METHODS_AND_TYPES, :predefined).
+      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
+      add(PREDEFINED_EXCEPTIONS, :exception)
+    
+    NAME = / [^\W\d] \w* /x
+    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
+    
+    OPERATOR = /
+      \.\.\. |          # ellipsis
+      \.(?!\d) |        # dot but not decimal point
+      [,;:()\[\]{}] |   # simple delimiters
+      \/\/=? | \*\*=? | # special math
+      [-+*\/%&|^]=? |   # ordinary math and binary logic
+      [~`] |            # binary complement and inspection
+      <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
+    /x
+    
+    STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
+      h[delimiter] = Regexp.union delimiter
+    end
+    
+    STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
+      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
+    end
+    
+    DEF_NEW_STATE = WordList.new(:initial).
+      add(%w(def), :def_expected).
+      add(%w(import from), :include_expected).
+      add(%w(class), :class_expected)
+    
+    DESCRIPTOR = /
+      #{NAME}
+      (?: \. #{NAME} )*
+      | \*
+    /x
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      string_delimiter = nil
+      string_raw = false
+      import_clause = class_name_follows = last_token_dot = false
+      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+      from_import_state = []
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        if state == :string
+          if scan(STRING_DELIMITER_REGEXP[string_delimiter])
+            tokens << [matched, :delimiter]
+            tokens << [:close, :string]
+            state = :initial
+            next
+          elsif string_delimiter.size == 3 && scan(/\n/)
+            kind = :content
+          elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
+            kind = :content
+          elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
+            kind = :char
+          elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
+            kind = :char
+          elsif scan(/ \\ . /x)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :string]
+            kind = :error
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
+          end
+        
+        elsif match = scan(/ [ \t]+ | \\\n /x)
+          tokens << [match, :space]
+          next
+        
+        elsif match = scan(/\n/)
+          tokens << [match, :space]
+          state = :initial if state == :include_expected
+          next
+        
+        elsif match = scan(/ \# [^\n]* /mx)
+          tokens << [match, :comment]
+          next
+        
+        elsif state == :initial
+          
+          if scan(/#{OPERATOR}/o)
+            kind = :operator
+          
+          elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+            tokens << [:open, :string]
+            string_delimiter = self[2]
+            string_raw = false
+            modifiers = self[1]
+            unless modifiers.empty?
+              string_raw = !!modifiers.index(?r)
+              tokens << [modifiers, :modifier]
+              match = string_delimiter
+            end
+            state = :string
+            kind = :delimiter
+          
+          # TODO: backticks
+          
+          elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+            kind = IDENT_KIND[match]
+            # TODO: keyword arguments
+            kind = :ident if last_token_dot
+            if kind == :old_keyword
+              kind = check(/\(/) ? :ident : :keyword
+            elsif kind == :predefined && check(/ *=/)
+              kind = :ident
+            elsif kind == :keyword
+              state = DEF_NEW_STATE[match]
+              from_import_state << match.to_sym if state == :include_expected
+            end
+          
+          elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
+            kind = :decorator
+          
+          elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
+            kind = :hex
+          
+          elsif scan(/0[bB][01]+[lL]?/)
+            kind = :bin
+          
+          elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+            kind = :float
+            if scan(/[jJ]/)
+              match << matched
+              kind = :imaginary
+            end
+          
+          elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
+            kind = :oct
+          
+          elsif match = scan(/\d+([lL])?/)
+            kind = :integer
+            if self[1] == nil && scan(/[jJ]/)
+              match << matched
+              kind = :imaginary
+            end
+          
+          else
+            getch
+            kind = :error
+          
+          end
+            
+        elsif state == :def_expected
+          state = :initial
+          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+            kind = :method
+          else
+            next
+          end
+        
+        elsif state == :class_expected
+          state = :initial
+          if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
+            kind = :class
+          else
+            next
+          end
+          
+        elsif state == :include_expected
+          if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
+            kind = :include
+            if match == 'as'
+              kind = :keyword
+              from_import_state << :as
+            elsif from_import_state.first == :from && match == 'import'
+              kind = :keyword
+              from_import_state << :import
+            elsif from_import_state.last == :as
+              # kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
+              kind = :ident
+              from_import_state.pop
+            elsif IDENT_KIND[match] == :keyword
+              unscan
+              match = nil
+              state = :initial
+              next
+            end
+          elsif match = scan(/,/)
+            from_import_state.pop if from_import_state.last == :as
+            kind = :operator
+          else
+            from_import_state = []
+            state = :initial
+            next
+          end
+          
+        else
+          raise_inspect 'Unknown state', tokens, state
+          
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens, state unless match
+        
+        last_token_dot = match == '.'
+        
+        tokens << [match, kind]
+        
+      end
+      
+      if state == :string
+        tokens << [:close, :string]
+      end
+      
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/rhtml.rb

@@ -9,6 +9,9 @@ module Scanners
 
     include Streamable
     register_for :rhtml
+    title 'HTML ERB Template'
+    
+    KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
 
     ERB_RUBY_BLOCK = /
       <%(?!%)[=-]?

data/lib/coderay/scanners/ruby.rb

@@ -21,6 +21,10 @@ module Scanners
     file_extension 'rb'
 
     helper :patterns
+    
+    if not defined? EncodingError
+      EncodingError = Class.new Exception
+    end
 
   private
     def scan_tokens tokens, options
@@ -31,9 +35,10 @@ module Scanners
       state = :initial
       depth = nil
       inline_block_stack = []
-
+      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+      
       patterns = Patterns  # avoid constant lookup
-
+      
       until eos?
         match = nil
         kind = nil
@@ -125,14 +130,14 @@ module Scanners
           if match = scan(/[ \t\f]+/)
             kind = :space
             match << scan(/\s*/) unless eos? || heredocs
-            value_expected = true if match.index(?\n) # FIXME not quite true
+            value_expected = true if match.index(?\n)
             tokens << [match, kind]
             next
             
           elsif match = scan(/\\?\n/)
             kind = :space
             if match == "\n"
-              value_expected = true  # FIXME not quite true
+              value_expected = true
               state = :initial if state == :undef_comma_expected
             end
             if heredocs
@@ -154,14 +159,14 @@ module Scanners
           elsif match = scan(/\#.*/) or
             ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
               kind = :comment
-              value_expected = true
               tokens << [match, kind]
               next
 
           elsif state == :initial
 
             # IDENTS #
-            if match = scan(/#{patterns::METHOD_NAME}/o)
+            if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
+                                      /#{patterns::METHOD_NAME}/o)
               if last_token_dot
                 kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
               else
@@ -170,9 +175,9 @@ module Scanners
                   kind = :constant
                 elsif kind == :reserved
                   state = patterns::DEF_NEW_STATE[match]
+                  value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
                 end
               end
-              ## experimental!
               value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
             
             elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
@@ -180,7 +185,6 @@ module Scanners
               value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
 
             # OPERATORS #
-            # TODO: match (), [], {} as one single operator
             elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
               if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
                 value_expected = :set
@@ -281,13 +285,24 @@ module Scanners
 
             else
               kind = :error
-              match = getch
+              match = (scan(/./mu) rescue nil) || getch
+              if !unicode && match.size > 1
+                # warn 'Switching to unicode mode because of char %p' % [match]
+                unicode = true
+                unscan
+                next
+              end
 
             end
 
           elsif state == :def_expected
             state = :initial
-            if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
+            if scan(/self\./)
+              tokens << ['self', :pre_constant]
+              tokens << ['.', :operator]
+            end
+            if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
+                                      /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
               kind = :method
             else
               next
@@ -327,7 +342,10 @@ module Scanners
             end
 
           elsif state == :alias_expected
-            if match = scan(/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+            match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
+                                   /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+            
+            if match
               tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
               tokens << [self[2], :space]
               tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]

data/lib/coderay/scanners/ruby/patterns.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 module CodeRay
 module Scanners
 
@@ -31,18 +32,18 @@ module Scanners
       add(RESERVED_WORDS, :reserved).
       add(PREDEFINED_CONSTANTS, :pre_constant)
 
-    IDENT = /[a-z_][\w_]*/i
+    IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
 
     METHOD_NAME = / #{IDENT} [?!]? /ox
     METHOD_NAME_OPERATOR = /
       \*\*?           # multiplication and power
-      | [-+~]@?       # plus, minus, tilde with and without @
-      | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
+      | [-+~]@?       # plus, minus, tilde with and without at sign
+      | [\/%&|^`]     # division, modulo or format strings, and, or, xor, system
       | \[\]=?        # array getter and setter
       | << | >>       # append or shift left, shift right
       | <=?>? | >=?   # comparison, rocket operator
       | ===? | =~     # simple equality, case equality, match
-      | ![~=@]?       # negation with and without @, not-equal and not-match
+      | ![~=@]?       # negation with and without at sign, not-equal and not-match
     /ox
     METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
     INSTANCE_VARIABLE = / @ #{IDENT} /ox
@@ -59,7 +60,7 @@ module Scanners
     QUOTE_TO_TYPE.default = :string
 
     REGEXP_MODIFIERS = /[mixounse]*/
-    REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/
+    REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
 
     DECIMAL = /\d+(?:_\d+)*/
     OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
@@ -141,14 +142,19 @@ module Scanners
       | #{CHARACTER}
       )
     /x
+    KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
+      and end in or unless begin
+      defined? ensure redo super until
+      break do next rescue then
+      when case else for retry
+      while elsif if not return
+      yield
+    ])
 
     RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
 
     RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
 
-    # FIXME: \s and = are only a workaround, they are still allowed
-    # as delimiters.
-    FANCY_START_SAVE = / % ( [qQwWxsr] | (?![a-zA-Z0-9\s=]) ) ([^a-zA-Z0-9]) /mx
     FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
 
     FancyStringType = {
@@ -174,15 +180,15 @@ module Scanners
       CLOSING_PAREN.each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
       OPENING_PAREN = CLOSING_PAREN.invert
 
-      STRING_PATTERN = Hash.new { |h, k|
+      STRING_PATTERN = Hash.new do |h, k|
         delim, interpreted = *k
-        delim_pattern = Regexp.escape(delim)
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
         if closing_paren = CLOSING_PAREN[delim]
           delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION  # JRuby fix
           delim_pattern << Regexp.escape(closing_paren)
         end
-
-
+        delim_pattern << '\\\\' unless delim == '\\'
+        
         special_escapes =
           case interpreted
           when :regexp_symbols
@@ -190,18 +196,18 @@ module Scanners
           when :words
             '| \s'
           end
-
+        
         h[k] =
           if interpreted and not delim == '#'
-            / (?= [#{delim_pattern}\\] | \# [{$@] #{special_escapes} ) /mx
+            / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
           else
-            / (?= [#{delim_pattern}\\] #{special_escapes} ) /mx
+            / (?= [#{delim_pattern}] #{special_escapes} ) /mx
           end
-      }
+      end
 
-      HEREDOC_PATTERN = Hash.new { |h, k|
+      HEREDOC_PATTERN = Hash.new do |h, k|
         delim, interpreted, indented = *k
-        delim_pattern = Regexp.escape(delim.dup)
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
         delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
         h[k] =
           if interpreted
@@ -209,12 +215,12 @@ module Scanners
           else
             / (?= #{delim_pattern}() | \\ ) /mx
           end
-      }
+      end
 
       def initialize kind, interpreted, delim, heredoc = false
         if heredoc
           pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
-          delim  = nil
+          delim = nil
         else
           pattern = STRING_PATTERN[ [delim, interpreted] ]
           if paren = CLOSING_PAREN[delim]