coderay 0.9.8 → 1.0.0

data/lib/coderay/scanners/ruby/patterns.rb

@@ -2,9 +2,9 @@
 module CodeRay
 module Scanners
 
-  module Ruby::Patterns  # :nodoc:
+  module Ruby::Patterns  # :nodoc: all
 
-    RESERVED_WORDS = %w[
+    KEYWORDS = %w[
       and def end in or unless begin
       defined? ensure module redo super until
       BEGIN break do next rescue then
@@ -13,25 +13,27 @@ module Scanners
       undef yield
     ]
 
-    DEF_KEYWORDS = %w[ def ]
-    UNDEF_KEYWORDS = %w[ undef ]
-    ALIAS_KEYWORDS = %w[ alias ]
-    MODULE_KEYWORDS = %w[ class module ]
-    DEF_NEW_STATE = WordList.new(:initial).
-      add(DEF_KEYWORDS, :def_expected).
-      add(UNDEF_KEYWORDS, :undef_expected).
-      add(ALIAS_KEYWORDS, :alias_expected).
-      add(MODULE_KEYWORDS, :module_expected)
-
+    # See http://murfy.de/ruby-constants.
     PREDEFINED_CONSTANTS = %w[
       nil true false self
-      DATA ARGV ARGF
+      DATA ARGV ARGF ENV
+      FALSE TRUE NIL
+      STDERR STDIN STDOUT
+      TOPLEVEL_BINDING
+      RUBY_COPYRIGHT RUBY_DESCRIPTION RUBY_ENGINE RUBY_PATCHLEVEL
+      RUBY_PLATFORM RUBY_RELEASE_DATE RUBY_REVISION RUBY_VERSION
       __FILE__ __LINE__ __ENCODING__
     ]
 
     IDENT_KIND = WordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_CONSTANTS, :pre_constant)
+      add(KEYWORDS, :keyword).
+      add(PREDEFINED_CONSTANTS, :predefined_constant)
+
+    KEYWORD_NEW_STATE = WordList.new(:initial).
+      add(%w[ def ], :def_expected).
+      add(%w[ undef ], :undef_expected).
+      add(%w[ alias ], :alias_expected).
+      add(%w[ class module ], :module_expected)
 
     IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
 
@@ -46,7 +48,9 @@ module Scanners
       | ===? | =~     # simple equality, case equality, match
       | ![~=@]?       # negation with and without at sign, not-equal and not-match
     /ox
-    METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
+    METHOD_SUFFIX = / (?: [?!] | = (?![~>]|=(?!>)) ) /x
+    METHOD_NAME_EX = / #{IDENT} #{METHOD_SUFFIX}? | #{METHOD_NAME_OPERATOR} /ox
+    METHOD_AFTER_DOT = / #{IDENT} [?!]? | #{METHOD_NAME_OPERATOR} /ox
     INSTANCE_VARIABLE = / @ #{IDENT} /ox
     CLASS_VARIABLE = / @@ #{IDENT} /ox
     OBJECT_VARIABLE = / @@? #{IDENT} /ox
@@ -60,8 +64,7 @@ module Scanners
     }
     QUOTE_TO_TYPE.default = :string
 
-    REGEXP_MODIFIERS = /[mixounse]*/
-    REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
+    REGEXP_MODIFIERS = /[mousenix]*/
 
     DECIMAL = /\d+(?:_\d+)*/
     OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
@@ -87,7 +90,7 @@ module Scanners
         [abefnrstv]
       |  [0-7]{1,3}
       | x[0-9A-Fa-f]{1,2}
-      | .?
+      | .
     /mx
     
     CONTROL_META_ESCAPE = /
@@ -110,12 +113,10 @@ module Scanners
 
     # NOTE: This is not completely correct, but
     # nobody needs heredoc delimiters ending with \n.
-    # Also, delimiters starting with numbers are allowed.
-    # but they are more often than not a false positive.
     HEREDOC_OPEN = /
       << (-)?              # $1 = float
       (?:
-        ( #{IDENT} )       # $2 = delim
+        ( [A-Za-z_0-9]+ )  # $2 = delim
       |
         ( ["'`\/] )        # $3 = quote, type
         ( [^\n]*? ) \3     # $4 = delim
@@ -134,6 +135,8 @@ module Scanners
       (?: \Z | (?=^\#CODE) )
     /mx
     
+    RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
+
     # Checks for a valid value to follow. This enables
     # value_expected in method calls without parentheses.
     VALUE_FOLLOWS = /
@@ -144,7 +147,7 @@ module Scanners
       | [-+] \d
       | #{CHARACTER}
       )
-    /x
+    /ox
     KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
       and end in or unless begin
       defined? ensure redo super until
@@ -153,89 +156,20 @@ module Scanners
       while elsif if not return
       yield
     ])
-
-    RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
-
-    RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
-
-    FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
-
-    FancyStringType = {
-      'q' => [:string, false],
-      'Q' => [:string, true],
-      'r' => [:regexp, true],
-      's' => [:symbol, false],
-      'x' => [:shell, true]
-    }
-    FancyStringType['w'] = FancyStringType['q']
-    FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
-
-    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
-      :paren, :paren_depth, :pattern, :next_state
-
-      CLOSING_PAREN = Hash[ *%w[
-        ( )
-        [ ]
-        < >
-        { }
-      ] ]
-
-      CLOSING_PAREN.each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
-      OPENING_PAREN = CLOSING_PAREN.invert
-
-      STRING_PATTERN = Hash.new do |h, k|
-        delim, interpreted = *k
-        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
-        if closing_paren = CLOSING_PAREN[delim]
-          delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION  # JRuby fix
-          delim_pattern << Regexp.escape(closing_paren)
-        end
-        delim_pattern << '\\\\' unless delim == '\\'
-        
-        special_escapes =
-          case interpreted
-          when :regexp_symbols
-            '| ' + REGEXP_SYMBOLS.source
-          when :words
-            '| \s'
-          end
-        
-        h[k] =
-          if interpreted and not delim == '#'
-            / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
-          else
-            / (?= [#{delim_pattern}] #{special_escapes} ) /mx
-          end
-      end
-
-      HEREDOC_PATTERN = Hash.new do |h, k|
-        delim, interpreted, indented = *k
-        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
-        delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
-        h[k] =
-          if interpreted
-            / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
-          else
-            / (?= #{delim_pattern}() | \\ ) /mx
-          end
-      end
-
-      def initialize kind, interpreted, delim, heredoc = false
-        if heredoc
-          pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
-          delim = nil
-        else
-          pattern = STRING_PATTERN[ [delim, interpreted] ]
-          if paren = CLOSING_PAREN[delim]
-            delim, paren = paren, delim
-            paren_depth = 1
-          end
-        end
-        super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
-      end
-    end unless defined? StringState
-
+    
+    FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
+    FANCY_STRING_KIND = Hash.new(:string).merge({
+      'r' => :regexp,
+      's' => :symbol,
+      'x' => :shell,
+    })
+    FANCY_STRING_INTERPRETED = Hash.new(true).merge({
+      'q' => false,
+      's' => false,
+      'w' => false,
+    })
+    
   end
-
+  
 end
 end

data/lib/coderay/scanners/ruby/string_state.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+module CodeRay
+module Scanners
+  
+  class Ruby
+    
+    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+      :opening_paren, :paren_depth, :pattern, :next_state  # :nodoc: all
+      
+      CLOSING_PAREN = Hash[ *%w[
+        ( )
+        [ ]
+        < >
+        { }
+      ] ].each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
+      
+      STRING_PATTERN = Hash.new do |h, k|
+        delim, interpreted = *k
+        # delim = delim.dup  # workaround for old Ruby
+        delim_pattern = Regexp.escape(delim)
+        if closing_paren = CLOSING_PAREN[delim]
+          delim_pattern << Regexp.escape(closing_paren)
+        end
+        delim_pattern << '\\\\' unless delim == '\\'
+        
+        # special_escapes =
+        #   case interpreted
+        #   when :regexp_symbols
+        #     '| [|?*+(){}\[\].^$]'
+        #   end
+        
+        h[k] =
+          if interpreted && delim != '#'
+            / (?= [#{delim_pattern}] | \# [{$@] ) /mx
+          else
+            / (?= [#{delim_pattern}] ) /mx
+          end
+      end
+      
+      def initialize kind, interpreted, delim, heredoc = false
+        if heredoc
+          pattern = heredoc_pattern delim, interpreted, heredoc == :indented
+          delim = nil
+        else
+          pattern = STRING_PATTERN[ [delim, interpreted] ]
+          if closing_paren = CLOSING_PAREN[delim]
+            opening_paren = delim
+            delim = closing_paren
+            paren_depth = 1
+          end
+        end
+        super kind, interpreted, delim, heredoc, opening_paren, paren_depth, pattern, :initial
+      end
+      
+      def heredoc_pattern delim, interpreted, indented
+        # delim = delim.dup  # workaround for old Ruby
+        delim_pattern = Regexp.escape(delim)
+        delim_pattern = / (?:\A|\n) #{ '(?>[ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+        if interpreted
+          / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
+        else
+          / (?= #{delim_pattern}() | \\ ) /mx
+        end
+      end
+      
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/sql.rb

@@ -5,31 +5,49 @@ module CodeRay module Scanners
 
     register_for :sql
     
-    RESERVED_WORDS = %w(
-      and as avg before begin between by case collate columns create database
-      databases delete distinct drop else end engine exists fields from full
-      group having if index inner insert into is join key like not on or order
-      outer primary prompt replace select set show table tables then trigger
-      union update using values when where
+    KEYWORDS = %w(
+      all and any as before begin between by case check collate
+      each else end exists
+      for foreign from full group having if in inner is join
+      like not of on or order outer over references
+      then to union using values when where
+      left right distinct
+    )
+    
+    OBJECTS = %w(
+      database databases table tables column columns fields index constraint
+      constraints transaction function procedure row key view trigger
+    )
+    
+    COMMANDS = %w(
+      add alter comment create delete drop grant insert into select update set
+      show prompt begin commit rollback replace truncate
     )
     
     PREDEFINED_TYPES = %w(
-      bigint bin binary bit blob bool boolean char date datetime decimal
-      double enum float hex int integer longblob longtext mediumblob mediumint
-      mediumtext oct smallint text time timestamp tinyblob tinyint tinytext
-      unsigned varchar year
+      char varchar varchar2 enum binary text tinytext mediumtext
+      longtext blob tinyblob mediumblob longblob timestamp
+      date time datetime year double decimal float int
+      integer tinyint mediumint bigint smallint unsigned bit
+      bool boolean hex bin oct
     )
     
-    PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
+    PREDEFINED_FUNCTIONS = %w( sum cast substring abs pi count min max avg now )
     
-    DIRECTIVES = %w( auto_increment unique default charset )
+    DIRECTIVES = %w( 
+      auto_increment unique default charset initially deferred
+      deferrable cascade immediate read write asc desc after
+      primary foreign return engine
+    )
     
     PREDEFINED_CONSTANTS = %w( null true false )
     
-    IDENT_KIND = CaseIgnoringWordList.new(:ident).
-      add(RESERVED_WORDS, :reserved).
-      add(PREDEFINED_TYPES, :pre_type).
-      add(PREDEFINED_CONSTANTS, :pre_constant).
+    IDENT_KIND = WordList::CaseIgnoring.new(:ident).
+      add(KEYWORDS, :keyword).
+      add(OBJECTS, :type).
+      add(COMMANDS, :class).
+      add(PREDEFINED_TYPES, :predefined_type).
+      add(PREDEFINED_CONSTANTS, :predefined_constant).
       add(PREDEFINED_FUNCTIONS, :predefined).
       add(DIRECTIVES, :directive)
     
@@ -38,58 +56,60 @@ module CodeRay module Scanners
     
     STRING_PREFIXES = /[xnb]|_\w+/i
     
-    def scan_tokens tokens, options
+    def scan_tokens encoder, options
       
       state = :initial
       string_type = nil
       string_content = ''
+      name_expected = false
       
       until eos?
         
-        kind = nil
-        match = nil
-        
         if state == :initial
           
-          if scan(/ \s+ | \\\n /x)
-            kind = :space
+          if match = scan(/ \s+ | \\\n /x)
+            encoder.text_token match, :space
           
-          elsif scan(/(?:--\s?|#).*/)
-            kind = :comment
+          elsif match = scan(/(?:--\s?|#).*/)
+            encoder.text_token match, :comment
             
-          elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
-            kind = :comment
+          elsif match = scan(%r( /\* (!)? (?: .*? \*/ | .* ) )mx)
+            encoder.text_token match, self[1] ? :directive : :comment
             
-          elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
-            kind = :operator
+          elsif match = scan(/ [*\/=<>:;,!&^|()\[\]{}~%] | [-+\.](?!\d) /x)
+            name_expected = true if match == '.' && check(/[A-Za-z_]/)
+            encoder.text_token match, :operator
             
-          elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
+          elsif match = scan(/(#{STRING_PREFIXES})?([`"'])/o)
             prefix = self[1]
             string_type = self[2]
-            tokens << [:open, :string]
-            tokens << [prefix, :modifier] if prefix
+            encoder.begin_group :string
+            encoder.text_token prefix, :modifier if prefix
             match = string_type
             state = :string
-            kind = :delimiter
+            encoder.text_token match, :delimiter
             
           elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
-            kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
+            encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
+            name_expected = false
             
-          elsif scan(/0[xX][0-9A-Fa-f]+/)
-            kind = :hex
+          elsif match = scan(/0[xX][0-9A-Fa-f]+/)
+            encoder.text_token match, :hex
             
-          elsif scan(/0[0-7]+(?![89.eEfF])/)
-            kind = :oct
+          elsif match = scan(/0[0-7]+(?![89.eEfF])/)
+            encoder.text_token match, :octal
             
-          elsif scan(/(?>\d+)(?![.eEfF])/)
-            kind = :integer
+          elsif match = scan(/[-+]?(?>\d+)(?![.eEfF])/)
+            encoder.text_token match, :integer
             
-          elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
-            kind = :float
+          elsif match = scan(/[-+]?(?:\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+)/)
+            encoder.text_token match, :float
+          
+          elsif match = scan(/\\N/)
+            encoder.text_token match, :predefined_constant
             
           else
-            getch
-            kind = :error
+            encoder.text_token getch, :error
             
           end
           
@@ -104,54 +124,48 @@ module CodeRay module Scanners
                 next
               end
               unless string_content.empty?
-                tokens << [string_content, :content]
+                encoder.text_token string_content, :content
                 string_content = ''
               end
-              tokens << [matched, :delimiter]
-              tokens << [:close, :string]
+              encoder.text_token match, :delimiter
+              encoder.end_group :string
               state = :initial
               string_type = nil
-              next
             else
               string_content << match
             end
-            next
-          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+          elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
             unless string_content.empty?
-              tokens << [string_content, :content]
+              encoder.text_token string_content, :content
               string_content = ''
             end
-            kind = :char
+            encoder.text_token match, :char
           elsif match = scan(/ \\ . /mox)
             string_content << match
             next
-          elsif scan(/ \\ | $ /x)
+          elsif match = scan(/ \\ | $ /x)
             unless string_content.empty?
-              tokens << [string_content, :content]
+              encoder.text_token string_content, :content
               string_content = ''
             end
-            kind = :error
+            encoder.text_token match, :error
             state = :initial
           else
-            raise "else case \" reached; %p not handled." % peek(1), tokens
+            raise "else case \" reached; %p not handled." % peek(1), encoder
           end
           
         else
-          raise 'else-case reached', tokens
+          raise 'else-case reached', encoder
           
         end
         
-        match ||= matched
-        unless kind
-          raise_inspect 'Error token %p in line %d' %
-            [[match, kind], line], tokens, state
-        end
-        raise_inspect 'Empty token', tokens unless match
-        
-        tokens << [match, kind]
-        
       end
-      tokens
+      
+      if state == :string
+        encoder.end_group state
+      end
+      
+      encoder
       
     end