coderay-beta 0.9.1

data/lib/coderay/scanners/ruby/patterns.rb

@@ -0,0 +1,238 @@
+# encoding: utf-8
+module CodeRay
+module Scanners
+
+  module Ruby::Patterns  # :nodoc:
+
+    RESERVED_WORDS = %w[
+      and def end in or unless begin
+      defined? ensure module redo super until
+      BEGIN break do next rescue then
+      when END case else for retry
+      while alias class elsif if not return
+      undef yield
+    ]
+
+    DEF_KEYWORDS = %w[ def ]
+    UNDEF_KEYWORDS = %w[ undef ]
+    ALIAS_KEYWORDS = %w[ alias ]
+    MODULE_KEYWORDS = %w[class module]
+    DEF_NEW_STATE = WordList.new(:initial).
+      add(DEF_KEYWORDS, :def_expected).
+      add(UNDEF_KEYWORDS, :undef_expected).
+      add(ALIAS_KEYWORDS, :alias_expected).
+      add(MODULE_KEYWORDS, :module_expected)
+
+    PREDEFINED_CONSTANTS = %w[
+      nil true false self
+      DATA ARGV ARGF __FILE__ __LINE__
+    ]
+
+    IDENT_KIND = WordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
+
+    IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
+
+    METHOD_NAME = / #{IDENT} [?!]? /ox
+    METHOD_NAME_OPERATOR = /
+      \*\*?           # multiplication and power
+      | [-+~]@?       # plus, minus, tilde with and without at sign
+      | [\/%&|^`]     # division, modulo or format strings, and, or, xor, system
+      | \[\]=?        # array getter and setter
+      | << | >>       # append or shift left, shift right
+      | <=?>? | >=?   # comparison, rocket operator
+      | ===? | =~     # simple equality, case equality, match
+      | ![~=@]?       # negation with and without at sign, not-equal and not-match
+    /ox
+    METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
+    INSTANCE_VARIABLE = / @ #{IDENT} /ox
+    CLASS_VARIABLE = / @@ #{IDENT} /ox
+    OBJECT_VARIABLE = / @@? #{IDENT} /ox
+    GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
+    VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
+
+    QUOTE_TO_TYPE = {
+      '`' => :shell,
+      '/'=> :regexp,
+    }
+    QUOTE_TO_TYPE.default = :string
+
+    REGEXP_MODIFIERS = /[mixounse]*/
+    REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
+
+    DECIMAL = /\d+(?:_\d+)*/
+    OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+    HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+    BINARY = /0b[01]+(?:_[01]+)*/
+
+    EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+    FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+    FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+    NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+
+    SYMBOL = /
+      :
+      (?:
+        #{METHOD_NAME_EX}
+      | #{PREFIX_VARIABLE}
+      | ['"]
+      )
+    /ox
+    METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
+
+    SIMPLE_ESCAPE = /
+        [abefnrstv]
+      |  [0-7]{1,3}
+      | x[0-9A-Fa-f]{1,2}
+      | .?
+    /mx
+    
+    CONTROL_META_ESCAPE = /
+      (?: M-|C-|c )
+      (?: \\ (?: M-|C-|c ) )*
+      (?: [^\\] | \\ #{SIMPLE_ESCAPE} )?
+    /mox
+    
+    ESCAPE = /
+      #{CONTROL_META_ESCAPE} | #{SIMPLE_ESCAPE}
+    /mox
+    
+    CHARACTER = /
+      \?
+      (?:
+        [^\s\\]
+      | \\ #{ESCAPE}
+      )
+    /mox
+
+    # NOTE: This is not completely correct, but
+    # nobody needs heredoc delimiters ending with \n.
+    HEREDOC_OPEN = /
+      << (-)?              # $1 = float
+      (?:
+        ( [A-Za-z_0-9]+ )  # $2 = delim
+      |
+        ( ["'`\/] )        # $3 = quote, type
+        ( [^\n]*? ) \3     # $4 = delim
+      )
+    /mx
+
+    RUBYDOC = /
+      =begin (?!\S)
+      .*?
+      (?: \Z | ^=end (?!\S) [^\n]* )
+    /mx
+
+    DATA = /
+      __END__$
+      .*?
+      (?: \Z | (?=^\#CODE) )
+    /mx
+    
+    # Checks for a valid value to follow. This enables
+    # value_expected in method calls without parentheses.
+    VALUE_FOLLOWS = /
+      (?>[ \t\f\v]+)
+      (?:
+        [%\/][^\s=]
+      | <<-?\S
+      | [-+] \d
+      | #{CHARACTER}
+      )
+    /x
+    KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
+      and end in or unless begin
+      defined? ensure redo super until
+      break do next rescue then
+      when case else for retry
+      while elsif if not return
+      yield
+    ])
+
+    RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
+
+    RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
+
+    FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
+
+    FancyStringType = {
+      'q' => [:string, false],
+      'Q' => [:string, true],
+      'r' => [:regexp, true],
+      's' => [:symbol, false],
+      'x' => [:shell, true]
+    }
+    FancyStringType['w'] = FancyStringType['q']
+    FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
+
+    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+      :paren, :paren_depth, :pattern, :next_state
+
+      CLOSING_PAREN = Hash[ *%w[
+        ( )
+        [ ]
+        < >
+        { }
+      ] ]
+
+      CLOSING_PAREN.each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
+      OPENING_PAREN = CLOSING_PAREN.invert
+
+      STRING_PATTERN = Hash.new do |h, k|
+        delim, interpreted = *k
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
+        if closing_paren = CLOSING_PAREN[delim]
+          delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION  # JRuby fix
+          delim_pattern << Regexp.escape(closing_paren)
+        end
+        delim_pattern << '\\\\' unless delim == '\\'
+        
+        special_escapes =
+          case interpreted
+          when :regexp_symbols
+            '| ' + REGEXP_SYMBOLS.source
+          when :words
+            '| \s'
+          end
+        
+        h[k] =
+          if interpreted and not delim == '#'
+            / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
+          else
+            / (?= [#{delim_pattern}] #{special_escapes} ) /mx
+          end
+      end
+
+      HEREDOC_PATTERN = Hash.new do |h, k|
+        delim, interpreted, indented = *k
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
+        delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+        h[k] =
+          if interpreted
+            / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
+          else
+            / (?= #{delim_pattern}() | \\ ) /mx
+          end
+      end
+
+      def initialize kind, interpreted, delim, heredoc = false
+        if heredoc
+          pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
+          delim = nil
+        else
+          pattern = STRING_PATTERN[ [delim, interpreted] ]
+          if paren = CLOSING_PAREN[delim]
+            delim, paren = paren, delim
+            paren_depth = 1
+          end
+        end
+        super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
+      end
+    end unless defined? StringState
+
+  end
+
+end
+end

data/lib/coderay/scanners/scheme.rb

@@ -0,0 +1,145 @@
+module CodeRay
+  module Scanners
+
+    # Scheme scanner for CodeRay (by closure).
+    # Thanks to murphy for putting CodeRay into public.
+    class Scheme < Scanner
+      
+      # TODO: function defs
+      # TODO: built-in functions
+      
+      register_for :scheme
+      file_extension 'scm'
+
+      CORE_FORMS = %w[
+        lambda let let* letrec syntax-case define-syntax let-syntax
+        letrec-syntax begin define quote if or and cond case do delay
+        quasiquote set! cons force call-with-current-continuation call/cc
+      ]
+
+      IDENT_KIND = CaseIgnoringWordList.new(:ident).
+        add(CORE_FORMS, :reserved)
+      
+      #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
+      #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
+      #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
+      IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
+      DIGIT = /\d/
+      DIGIT10 = DIGIT
+      DIGIT16 = /[0-9a-f]/i
+      DIGIT8 = /[0-7]/
+      DIGIT2 = /[01]/
+      RADIX16 = /\#x/i
+      RADIX8 = /\#o/i
+      RADIX2 = /\#b/i
+      RADIX10 = /\#d/i
+      EXACTNESS = /#i|#e/i
+      SIGN = /[\+-]?/
+      EXP_MARK = /[esfdl]/i
+      EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
+      SUFFIX = /#{EXP}?/
+      PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
+      PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
+      PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
+      PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
+      UINT10 = /#{DIGIT10}+#*/
+      UINT16 = /#{DIGIT16}+#*/
+      UINT8 = /#{DIGIT8}+#*/
+      UINT2 = /#{DIGIT2}+#*/
+      DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
+      UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
+      UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
+      UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
+      UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
+      REAL10 = /#{SIGN}#{UREAL10}/
+      REAL16 = /#{SIGN}#{UREAL16}/
+      REAL8 = /#{SIGN}#{UREAL8}/
+      REAL2 = /#{SIGN}#{UREAL2}/
+      IMAG10 = /i|#{UREAL10}i/
+      IMAG16 = /i|#{UREAL16}i/
+      IMAG8 = /i|#{UREAL8}i/
+      IMAG2 = /i|#{UREAL2}i/
+      COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
+      COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
+      COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
+      COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
+      NUM10 = /#{PREFIX10}?#{COMPLEX10}/
+      NUM16 = /#{PREFIX16}#{COMPLEX16}/
+      NUM8 = /#{PREFIX8}#{COMPLEX8}/
+      NUM2 = /#{PREFIX2}#{COMPLEX2}/
+      NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
+    
+    private
+      def scan_tokens tokens,options
+        
+        state = :initial
+        ident_kind = IDENT_KIND
+        
+        until eos?
+          kind = match = nil
+          
+          case state
+          when :initial
+            if scan(/ \s+ | \\\n /x)
+              kind = :space
+            elsif scan(/['\(\[\)\]]|#\(/)
+              kind = :operator_fat
+            elsif scan(/;.*/)
+              kind = :comment
+            elsif scan(/#\\(?:newline|space|.?)/)
+              kind = :char
+            elsif scan(/#[ft]/)
+              kind = :pre_constant
+            elsif scan(/#{IDENTIFIER}/o)
+              kind = ident_kind[matched]
+            elsif scan(/\./)
+              kind = :operator
+            elsif scan(/"/)
+              tokens << [:open, :string]
+              state = :string
+              tokens << ['"', :delimiter]
+              next
+            elsif scan(/#{NUM}/o) and not matched.empty?
+              kind = :integer
+            elsif getch
+              kind = :error
+            end
+            
+          when :string
+            if scan(/[^"\\]+/) or scan(/\\.?/)
+              kind = :content
+            elsif scan(/"/)
+              tokens << ['"', :delimiter]
+              tokens << [:close, :string]
+              state = :initial
+              next
+            else
+              raise_inspect "else case \" reached; %p not handled." % peek(1),
+                tokens, state
+            end
+            
+          else
+            raise "else case reached"
+          end
+          
+          match ||= matched
+          if $DEBUG and not kind
+            raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+          end
+          raise_inspect 'Empty token', tokens, state unless match
+          
+          tokens << [match, kind]
+          
+        end  # until eos
+        
+        if state == :string
+          tokens << [:close, :string]
+        end
+        
+        tokens
+        
+      end #scan_tokens
+    end #class
+  end #module scanners
+end #module coderay

data/lib/coderay/scanners/sql.rb

@@ -0,0 +1,162 @@
+module CodeRay module Scanners
+  
+  # by Josh Goebel
+  class SQL < Scanner
+
+    register_for :sql
+    
+    RESERVED_WORDS = %w(
+      create database table index trigger drop primary key set select
+      insert update delete replace into
+      on from values before and or if exists case when
+      then else as group order by avg where
+      join inner outer union engine not
+      like end using collate show columns begin
+    )
+    
+    PREDEFINED_TYPES = %w(
+      char varchar enum binary text tinytext mediumtext
+      longtext blob tinyblob mediumblob longblob timestamp
+      date time datetime year double decimal float int
+      integer tinyint mediumint bigint smallint unsigned bit
+      bool boolean hex bin oct
+    )
+    
+    PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
+    
+    DIRECTIVES = %w( auto_increment unique default charset )
+
+    PREDEFINED_CONSTANTS = %w( null true false )
+    
+    IDENT_KIND = CaseIgnoringWordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
+      add(PREDEFINED_CONSTANTS, :pre_constant).
+      add(PREDEFINED_FUNCTIONS, :predefined).
+      add(DIRECTIVES, :directive)
+    
+    ESCAPE = / [rbfntv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+    
+    STRING_PREFIXES = /[xnb]|_\w+/i
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      string_type = nil
+      string_content = ''
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        if state == :initial
+          
+          if scan(/ \s+ | \\\n /x)
+            kind = :space
+          
+          elsif scan(/^(?:--\s?|#).*/)
+            kind = :comment
+            
+          elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
+            kind = :comment
+            
+          elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
+            kind = :operator
+            
+          elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
+            prefix = self[1]
+            string_type = self[2]
+            tokens << [:open, :string]
+            tokens << [prefix, :modifier] if prefix
+            match = string_type
+            state = :string
+            kind = :delimiter
+            
+          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
+            
+          elsif scan(/0[xX][0-9A-Fa-f]+/)
+            kind = :hex
+            
+          elsif scan(/0[0-7]+(?![89.eEfF])/)
+            kind = :oct
+            
+          elsif scan(/(?>\d+)(?![.eEfF])/)
+            kind = :integer
+            
+          elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+            kind = :float
+            
+          else
+            getch
+            kind = :error
+            
+          end
+          
+        elsif state == :string
+          if match = scan(/[^\\"'`]+/)
+            string_content << match
+            next
+          elsif match = scan(/["'`]/)
+            if string_type == match
+              if peek(1) == string_type  # doubling means escape
+                string_content << string_type << getch
+                next
+              end
+              unless string_content.empty?
+                tokens << [string_content, :content]
+                string_content = ''
+              end
+              tokens << [matched, :delimiter]
+              tokens << [:close, :string]
+              state = :initial
+              string_type = nil
+              next
+            else
+              string_content << match
+            end
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            unless string_content.empty?
+              tokens << [string_content, :content]
+              string_content = ''
+            end
+            kind = :char
+          elsif match = scan(/ \\ . /mox)
+            string_content << match
+            next
+          elsif scan(/ \\ | $ /x)
+            unless string_content.empty?
+              tokens << [string_content, :content]
+              string_content = ''
+            end
+            kind = :error
+            state = :initial
+          else
+            raise "else case \" reached; %p not handled." % peek(1), tokens
+          end
+          
+        else
+          raise 'else-case reached', tokens
+          
+        end
+        
+        match ||= matched
+        unless kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+        
+      end
+      tokens
+      
+    end
+    
+  end
+  
+end end