raldred-coderay 0.9.0 → 0.9.339

data/lib/coderay/scanners/plaintext.rb

@@ -0,0 +1,18 @@
+module CodeRay
+module Scanners
+
+  class Plaintext < Scanner
+
+    register_for :plaintext, :plain
+    
+    include Streamable
+
+    def scan_tokens tokens, options
+      text = (scan_until(/\z/) || '')
+      tokens << [text, :plain]
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/python.rb

@@ -0,0 +1,232 @@
+module CodeRay
+module Scanners
+  
+  # Bases on pygments' PythonLexer, see
+  # http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
+  class Python < Scanner
+    
+    include Streamable
+    
+    register_for :python
+    file_extension 'py'
+    
+    KEYWORDS = [
+      'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
+      'del', 'elif', 'else', 'except', 'finally', 'for',
+      'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
+      'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
+      'nonlocal',  # new in Python 3
+    ]
+    
+    OLD_KEYWORDS = [
+      'exec', 'print',  # gone in Python 3
+    ]
+    
+    PREDEFINED_METHODS_AND_TYPES = %w[
+      __import__ abs all any apply basestring bin bool buffer
+      bytearray bytes callable chr classmethod cmp coerce compile
+      complex delattr dict dir divmod enumerate eval execfile exit
+      file filter float frozenset getattr globals hasattr hash hex id
+      input int intern isinstance issubclass iter len list locals
+      long map max min next object oct open ord pow property range
+      raw_input reduce reload repr reversed round set setattr slice
+      sorted staticmethod str sum super tuple type unichr unicode
+      vars xrange zip
+    ]
+    
+    PREDEFINED_EXCEPTIONS = %w[
+      ArithmeticError AssertionError AttributeError
+      BaseException DeprecationWarning EOFError EnvironmentError
+      Exception FloatingPointError FutureWarning GeneratorExit IOError
+      ImportError ImportWarning IndentationError IndexError KeyError
+      KeyboardInterrupt LookupError MemoryError NameError
+      NotImplemented NotImplementedError OSError OverflowError
+      OverflowWarning PendingDeprecationWarning ReferenceError
+      RuntimeError RuntimeWarning StandardError StopIteration
+      SyntaxError SyntaxWarning SystemError SystemExit TabError
+      TypeError UnboundLocalError UnicodeDecodeError
+      UnicodeEncodeError UnicodeError UnicodeTranslateError
+      UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
+    ]
+    
+    PREDEFINED_VARIABLES_AND_CONSTANTS = [
+      'False', 'True', 'None', # "keywords" since Python 3
+      'self', 'Ellipsis', 'NotImplemented',
+    ]
+    
+    IDENT_KIND = WordList.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(OLD_KEYWORDS, :old_keyword).
+      add(PREDEFINED_METHODS_AND_TYPES, :predefined).
+      add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
+      add(PREDEFINED_EXCEPTIONS, :exception)
+    
+    ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
+    
+    OPERATOR = /
+      \.\.\. |          # ellipsis
+      \.(?!\d) |        # dot but not decimal point
+      [,;:()\[\]{}] |   # simple delimiters
+      \/\/=? | \*\*=? | # special math
+      [-+*\/%&|^]=? |   # ordinary math and binary logic
+      ~ |               # binary complement
+      <<=? | >>=? | [<>=]=? | !=  # comparison and assignment
+    /x
+    
+    STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
+      h[delimiter] = Regexp.union delimiter
+    end
+    
+    STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
+      h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
+    end
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      string_delimiter = nil
+      import_clause = class_name_follows = last_token_dot = false
+      unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        case state
+        
+        when :initial
+          
+          if match = scan(/ [ \t]+ | \\?\n /x)
+            tokens << [match, :space]
+            next
+          
+          elsif match = scan(/ \# [^\n]* /mx)
+            tokens << [match, :comment]
+            next
+          
+          elsif scan(/#{OPERATOR}/o)
+            kind = :operator
+          
+          elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
+            tokens << [:open, :string]
+            string_delimiter = self[2]
+            string_raw = false
+            modifiers = self[1]
+            unless modifiers.empty?
+              string_raw = !!modifiers.index(?r)
+              tokens << [modifiers, :modifier]
+              match = string_delimiter
+            end
+            state = :string
+            kind = :delimiter
+          
+          elsif match = (unicode && scan(/[[:alpha:]_]\w*/ux)) ||
+                                    scan(/[[:alpha:]_]\w*/x)
+            kind = IDENT_KIND[match]
+            # TODO: handle class, def, from, import
+            # TODO: keyword arguments
+            kind = :ident if last_token_dot
+            kind = check(/\(/) ? :ident : :keyword if kind == :old_keyword
+          
+          elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
+            kind = :decorator
+          
+          elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
+            kind = :hex
+          
+          elsif scan(/0[bB][01]+[lL]?/)
+            kind = :bin
+          
+          elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+            kind = :float
+            if scan(/[jJ]/)
+              match << matched
+              kind = :imaginary
+            end
+          
+          elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
+            kind = :oct
+          
+          elsif match = scan(/\d+([lL])?/)
+            kind = :integer
+            if self[1] == nil && scan(/[jJ]/)
+              match << matched
+              kind = :imaginary
+            end
+          
+          else
+            getch
+            kind = :error
+          
+          end
+        
+        when :string
+          if scan(STRING_DELIMITER_REGEXP[string_delimiter])
+            tokens << [matched, :delimiter]
+            tokens << [:close, :string]
+            state = :initial
+            next
+          elsif string_delimiter.size == 3 && scan(/\n/)
+            kind = :content
+          elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
+            kind = :content
+          elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
+            kind = :char
+          elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
+            kind = :char
+          elsif scan(/ \\ . /x)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :string]
+            kind = :error
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
+          end
+        
+        when :include_expected
+          if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+            kind = :include
+            state = :initial
+          
+          elsif match = scan(/\s+/)
+            kind = :space
+            state = :initial if match.index ?\n
+          
+          else
+            getch
+            kind = :error
+          
+          end
+          
+        else
+          raise_inspect 'Unknown state', tokens, state
+        
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens, state unless match
+        
+        last_token_dot = match == '.'
+        
+        tokens << [match, kind]
+        
+      end
+      
+      if state == :string
+        tokens << [:close, :string]
+      end
+      
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/rhtml.rb

@@ -0,0 +1,71 @@
+module CodeRay
+module Scanners
+
+  load :html
+  load :ruby
+
+  # RHTML Scanner
+  class RHTML < Scanner
+
+    include Streamable
+    register_for :rhtml
+
+    ERB_RUBY_BLOCK = /
+      <%(?!%)[=-]?
+      (?>
+        [^\-%]*    # normal*
+        (?>        # special
+          (?: %(?!>) | -(?!%>) )
+          [^\-%]*  # normal*
+        )*
+      )
+      (?: -?%> )?
+    /x
+
+    START_OF_ERB = /
+      <%(?!%)
+    /x
+
+  private
+
+    def setup
+      @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+      @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+    end
+
+    def reset_instance
+      super
+      @html_scanner.reset
+    end
+
+    def scan_tokens tokens, options
+
+      until eos?
+
+        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
+          @html_scanner.tokenize match
+
+        elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
+          start_tag = match[/\A<%[-=]?/]
+          end_tag = match[/-?%?>?\z/]
+          tokens << [:open, :inline]
+          tokens << [start_tag, :inline_delimiter]
+          code = match[start_tag.size .. -1 - end_tag.size]
+          @ruby_scanner.tokenize code
+          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
+          tokens << [:close, :inline]
+
+        else
+          raise_inspect 'else-case reached!', tokens
+        end
+
+      end
+
+      tokens
+
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/ruby.rb

@@ -0,0 +1,386 @@
+module CodeRay
+module Scanners
+
+  # This scanner is really complex, since Ruby _is_ a complex language!
+  #
+  # It tries to highlight 100% of all common code,
+  # and 90% of strange codes.
+  #
+  # It is optimized for HTML highlighting, and is not very useful for
+  # parsing or pretty printing.
+  #
+  # For now, I think it's better than the scanners in VIM or Syntax, or
+  # any highlighter I was able to find, except Caleb's RubyLexer.
+  #
+  # I hope it's also better than the rdoc/irb lexer.
+  class Ruby < Scanner
+
+    include Streamable
+
+    register_for :ruby
+    file_extension 'rb'
+
+    helper :patterns
+
+  private
+    def scan_tokens tokens, options
+      last_token_dot = false
+      value_expected = true
+      heredocs = nil
+      last_state = nil
+      state = :initial
+      depth = nil
+      inline_block_stack = []
+
+      patterns = Patterns  # avoid constant lookup
+
+      until eos?
+        match = nil
+        kind = nil
+
+        if state.instance_of? patterns::StringState
+# {{{
+          match = scan_until(state.pattern) || scan_until(/\z/)
+          tokens << [match, :content] unless match.empty?
+          break if eos?
+
+          if state.heredoc and self[1]  # end of heredoc
+            match = getch.to_s
+            match << scan_until(/$/) unless eos?
+            tokens << [match, :delimiter]
+            tokens << [:close, state.type]
+            state = state.next_state
+            next
+          end
+
+          case match = getch
+
+          when state.delim
+            if state.paren
+              state.paren_depth -= 1
+              if state.paren_depth > 0
+                tokens << [match, :nesting_delimiter]
+                next
+              end
+            end
+            tokens << [match, :delimiter]
+            if state.type == :regexp and not eos?
+              modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
+              tokens << [modifiers, :modifier] unless modifiers.empty?
+            end
+            tokens << [:close, state.type]
+            value_expected = false
+            state = state.next_state
+
+          when '\\'
+            if state.interpreted
+              if esc = scan(/ #{patterns::ESCAPE} /ox)
+                tokens << [match + esc, :char]
+              else
+                tokens << [match, :error]
+              end
+            else
+              case m = getch
+              when state.delim, '\\'
+                tokens << [match + m, :char]
+              when nil
+                tokens << [match, :error]
+              else
+                tokens << [match + m, :content]
+              end
+            end
+
+          when '#'
+            case peek(1)
+            when '{'
+              inline_block_stack << [state, depth, heredocs]
+              value_expected = true
+              state = :initial
+              depth = 1
+              tokens << [:open, :inline]
+              tokens << [match + getch, :inline_delimiter]
+            when '$', '@'
+              tokens << [match, :escape]
+              last_state = state  # scan one token as normal code, then return here
+              state = :initial
+            else
+              raise_inspect 'else-case # reached; #%p not handled' % peek(1), tokens
+            end
+
+          when state.paren
+            state.paren_depth += 1
+            tokens << [match, :nesting_delimiter]
+
+          when /#{patterns::REGEXP_SYMBOLS}/ox
+            tokens << [match, :function]
+
+          else
+            raise_inspect 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
+
+          end
+          next
+# }}}
+        else
+# {{{
+          if match = scan(/[ \t\f]+/)
+            kind = :space
+            match << scan(/\s*/) unless eos? || heredocs
+            value_expected = true if match.index(?\n) # FIXME not quite true
+            tokens << [match, kind]
+            next
+            
+          elsif match = scan(/\\?\n/)
+            kind = :space
+            if match == "\n"
+              value_expected = true  # FIXME not quite true
+              state = :initial if state == :undef_comma_expected
+            end
+            if heredocs
+              unscan  # heredoc scanning needs \n at start
+              state = heredocs.shift
+              tokens << [:open, state.type]
+              heredocs = nil if heredocs.empty?
+              next
+            else
+              match << scan(/\s*/) unless eos?
+            end
+            tokens << [match, kind]
+            next
+          
+          elsif bol? && match = scan(/\#!.*/)
+            tokens << [match, :doctype]
+            next
+            
+          elsif match = scan(/\#.*/) or
+            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
+              kind = :comment
+              value_expected = true
+              tokens << [match, kind]
+              next
+
+          elsif state == :initial
+
+            # IDENTS #
+            if match = scan(/#{patterns::METHOD_NAME}/o)
+              if last_token_dot
+                kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
+              else
+                kind = patterns::IDENT_KIND[match]
+                if kind == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
+                  kind = :constant
+                elsif kind == :reserved
+                  state = patterns::DEF_NEW_STATE[match]
+                end
+              end
+              ## experimental!
+              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+            
+            elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o)
+              kind = :ident
+              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
+
+            # OPERATORS #
+            # TODO: match (), [], {} as one single operator
+            elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
+              if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
+                value_expected = :set
+              end
+              last_token_dot = :set if self[1]
+              kind = :operator
+              unless inline_block_stack.empty?
+                case match
+                when '{'
+                  depth += 1
+                when '}'
+                  depth -= 1
+                  if depth == 0  # closing brace of inline block reached
+                    state, depth, heredocs = inline_block_stack.pop
+                    heredocs = nil if heredocs && heredocs.empty?
+                    tokens << [match, :inline_delimiter]
+                    kind = :inline
+                    match = :close
+                  end
+                end
+              end
+
+            elsif match = scan(/ ['"] /mx)
+              tokens << [:open, :string]
+              kind = :delimiter
+              state = patterns::StringState.new :string, match == '"', match  # important for streaming
+
+            elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
+              kind = :instance_variable
+
+            elsif value_expected and match = scan(/\//)
+              tokens << [:open, :regexp]
+              kind = :delimiter
+              interpreted = true
+              state = patterns::StringState.new :regexp, interpreted, match
+
+            # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
+            elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
+              kind = self[1] ? :float : :integer
+
+            elsif match = scan(/#{patterns::SYMBOL}/o)
+              case delim = match[1]
+              when ?', ?"
+                tokens << [:open, :symbol]
+                tokens << [':', :symbol]
+                match = delim.chr
+                kind = :delimiter
+                state = patterns::StringState.new :symbol, delim == ?", match
+              else
+                kind = :symbol
+              end
+
+            elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
+              value_expected = :set
+              kind = :operator
+
+            elsif value_expected and match = scan(/#{patterns::HEREDOC_OPEN}/o)
+              indented = self[1] == '-'
+              quote = self[3]
+              delim = self[quote ? 4 : 2]
+              kind = patterns::QUOTE_TO_TYPE[quote]
+              tokens << [:open, kind]
+              tokens << [match, :delimiter]
+              match = :close
+              heredoc = patterns::StringState.new kind, quote != '\'', delim, (indented ? :indented : :linestart )
+              heredocs ||= []  # create heredocs if empty
+              heredocs << heredoc
+
+            elsif value_expected and match = scan(/#{patterns::FANCY_START_CORRECT}/o)
+              kind, interpreted = *patterns::FancyStringType.fetch(self[1]) do
+                raise_inspect 'Unknown fancy string: %%%p' % k, tokens
+              end
+              tokens << [:open, kind]
+              state = patterns::StringState.new kind, interpreted, self[2]
+              kind = :delimiter
+
+            elsif value_expected and match = scan(/#{patterns::CHARACTER}/o)
+              kind = :integer
+
+            elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
+              value_expected = :set
+              kind = :operator
+
+            elsif match = scan(/`/)
+              if last_token_dot
+                kind = :operator
+              else
+                tokens << [:open, :shell]
+                kind = :delimiter
+                state = patterns::StringState.new :shell, true, match
+              end
+
+            elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
+              kind = :global_variable
+
+            elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
+              kind = :class_variable
+
+            else
+              kind = :error
+              match = getch
+
+            end
+
+          elsif state == :def_expected
+            state = :initial
+            if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
+              kind = :method
+            else
+              next
+            end
+
+          elsif state == :module_expected
+            if match = scan(/<</)
+              kind = :operator
+            else
+              state = :initial
+              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
+                kind = :class
+              else
+                next
+              end
+            end
+
+          elsif state == :undef_expected
+            state = :undef_comma_expected
+            if match = scan(/#{patterns::METHOD_NAME_EX}/o)
+              kind = :method
+            elsif match = scan(/#{patterns::SYMBOL}/o)
+              case delim = match[1]
+              when ?', ?"
+                tokens << [:open, :symbol]
+                tokens << [':', :symbol]
+                match = delim.chr
+                kind = :delimiter
+                state = patterns::StringState.new :symbol, delim == ?", match
+                state.next_state = :undef_comma_expected
+              else
+                kind = :symbol
+              end
+            else
+              state = :initial
+              next
+            end
+
+          elsif state == :alias_expected
+            if match = scan(/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+              tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
+              tokens << [self[2], :space]
+              tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
+            end
+            state = :initial
+            next
+
+          elsif state == :undef_comma_expected
+            if match = scan(/,/)
+              kind = :operator
+              state = :undef_expected
+            else
+              state = :initial
+              next
+            end
+
+          end
+# }}}
+          
+          unless kind == :error
+            value_expected = value_expected == :set
+            last_token_dot = last_token_dot == :set
+          end
+          
+          if $DEBUG and not kind
+            raise_inspect 'Error token %p in line %d' %
+              [[match, kind], line], tokens, state
+          end
+          raise_inspect 'Empty token', tokens unless match
+
+          tokens << [match, kind]
+
+          if last_state
+            state = last_state
+            last_state = nil
+          end
+        end
+      end
+
+      inline_block_stack << [state] if state.is_a? patterns::StringState
+      until inline_block_stack.empty?
+        this_block = inline_block_stack.pop
+        tokens << [:close, :inline] if this_block.size > 1
+        state = this_block.first
+        tokens << [:close, state.type]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end
+
+# vim:fdm=marker