coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay/scanners/ruby/patterns.rb

@@ -4,7 +4,7 @@ module Scanners
 
   module Ruby::Patterns  # :nodoc: all
 
-    KEYWORDS = %w[
+    RESERVED_WORDS = %w[
       and def end in or unless begin
       defined? ensure module redo super until
       BEGIN break do next rescue then
@@ -26,8 +26,8 @@ module Scanners
     ]
 
     IDENT_KIND = WordList.new(:ident).
-      add(KEYWORDS, :keyword).
-      add(PREDEFINED_CONSTANTS, :predefined_constant)
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_CONSTANTS, :pre_constant)
 
     KEYWORD_NEW_STATE = WordList.new(:initial).
       add(%w[ def ], :def_expected).
@@ -65,6 +65,7 @@ module Scanners
     QUOTE_TO_TYPE.default = :string
 
     REGEXP_MODIFIERS = /[mousenix]*/
+    REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
 
     DECIMAL = /\d+(?:_\d+)*/
     OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
@@ -90,7 +91,7 @@ module Scanners
         [abefnrstv]
       |  [0-7]{1,3}
       | x[0-9A-Fa-f]{1,2}
-      | .
+      | .?
     /mx
     
     CONTROL_META_ESCAPE = /
@@ -156,20 +157,86 @@ module Scanners
       while elsif if not return
       yield
     ])
-    
-    FANCY_STRING_START = / % ( [QqrsWwx] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /x
-    FANCY_STRING_KIND = Hash.new(:string).merge({
-      'r' => :regexp,
-      's' => :symbol,
-      'x' => :shell,
-    })
-    FANCY_STRING_INTERPRETED = Hash.new(true).merge({
-      'q' => false,
-      's' => false,
-      'w' => false,
-    })
-    
+
+    FANCY_START = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
+
+    FancyStringType = {
+      'q' => [:string, false],
+      'Q' => [:string, true],
+      'r' => [:regexp, true],
+      's' => [:symbol, false],
+      'x' => [:shell, true]
+    }
+    FancyStringType['w'] = FancyStringType['q']
+    FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
+
+    class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+      :opening_paren, :paren_depth, :pattern, :next_state
+
+      CLOSING_PAREN = Hash[ *%w[
+        ( )
+        [ ]
+        < >
+        { }
+      ] ]
+
+      CLOSING_PAREN.each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
+      OPENING_PAREN = CLOSING_PAREN.invert
+
+      STRING_PATTERN = Hash.new do |h, k|
+        delim, interpreted = *k
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
+        if closing_paren = CLOSING_PAREN[delim]
+          delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION  # JRuby fix
+          delim_pattern << Regexp.escape(closing_paren)
+        end
+        delim_pattern << '\\\\' unless delim == '\\'
+        
+        special_escapes =
+          case interpreted
+          when :regexp_symbols
+            '| ' + REGEXP_SYMBOLS.source
+          when :words
+            '| \s'
+          end
+        
+        h[k] =
+          if interpreted and not delim == '#'
+            / (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
+          else
+            / (?= [#{delim_pattern}] #{special_escapes} ) /mx
+          end
+      end
+
+      HEREDOC_PATTERN = Hash.new do |h, k|
+        delim, interpreted, indented = *k
+        delim_pattern = Regexp.escape(delim.dup)  # dup: workaround for old Ruby
+        delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+        h[k] =
+          if interpreted
+            / (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
+          else
+            / (?= #{delim_pattern}() | \\ ) /mx
+          end
+      end
+
+      def initialize kind, interpreted, delim, heredoc = false
+        if heredoc
+          pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
+          delim = nil
+        else
+          pattern = STRING_PATTERN[ [delim, interpreted] ]
+          if closing_paren = CLOSING_PAREN[delim]
+            opening_paren = delim
+            delim = closing_paren
+            paren_depth = 1
+          end
+        end
+        super kind, interpreted, delim, heredoc, opening_paren, paren_depth, pattern, :initial
+      end
+    end unless defined? StringState
+
   end
-  
+
 end
 end

data/lib/coderay/scanners/scheme.rb

@@ -0,0 +1,136 @@
+module CodeRay
+  module Scanners
+
+    # Scheme scanner for CodeRay (by closure).
+    # 
+    # Thanks to murphy for putting CodeRay into public.
+    class Scheme < Scanner
+      
+      # TODO: function defs
+      # TODO: built-in functions
+      
+      register_for :scheme
+      file_extension 'scm'
+
+      CORE_FORMS = %w[
+        lambda let let* letrec syntax-case define-syntax let-syntax
+        letrec-syntax begin define quote if or and cond case do delay
+        quasiquote set! cons force call-with-current-continuation call/cc
+      ]  # :nodoc:
+      
+      IDENT_KIND = CaseIgnoringWordList.new(:ident).
+        add(CORE_FORMS, :reserved)  # :nodoc:
+      
+      #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
+      #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
+      #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
+      IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./  # :nodoc:
+      DIGIT = /\d/  # :nodoc:
+      DIGIT10 = /\d/  # :nodoc:
+      DIGIT16 = /[0-9a-f]/i  # :nodoc:
+      DIGIT8 = /[0-7]/  # :nodoc:
+      DIGIT2 = /[01]/  # :nodoc:
+      RADIX16 = /\#x/i  # :nodoc:
+      RADIX8 = /\#o/i  # :nodoc:
+      RADIX2 = /\#b/i  # :nodoc:
+      RADIX10 = /\#d/i  # :nodoc:
+      EXACTNESS = /#i|#e/i  # :nodoc:
+      SIGN = /[\+-]?/  # :nodoc:
+      EXP_MARK = /[esfdl]/i  # :nodoc:
+      EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/  # :nodoc:
+      SUFFIX = /#{EXP}?/  # :nodoc:
+      PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/  # :nodoc:
+      PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/  # :nodoc:
+      PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/  # :nodoc:
+      PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/  # :nodoc:
+      UINT10 = /#{DIGIT10}+#*/  # :nodoc:
+      UINT16 = /#{DIGIT16}+#*/  # :nodoc:
+      UINT8 = /#{DIGIT8}+#*/  # :nodoc:
+      UINT2 = /#{DIGIT2}+#*/  # :nodoc:
+      DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/  # :nodoc:
+      UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/  # :nodoc:
+      UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/  # :nodoc:
+      UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/  # :nodoc:
+      UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/  # :nodoc:
+      REAL10 = /#{SIGN}#{UREAL10}/  # :nodoc:
+      REAL16 = /#{SIGN}#{UREAL16}/  # :nodoc:
+      REAL8 = /#{SIGN}#{UREAL8}/  # :nodoc:
+      REAL2 = /#{SIGN}#{UREAL2}/  # :nodoc:
+      IMAG10 = /i|#{UREAL10}i/  # :nodoc:
+      IMAG16 = /i|#{UREAL16}i/  # :nodoc:
+      IMAG8 = /i|#{UREAL8}i/  # :nodoc:
+      IMAG2 = /i|#{UREAL2}i/  # :nodoc:
+      COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/  # :nodoc:
+      COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/  # :nodoc:
+      COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/  # :nodoc:
+      COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/  # :nodoc:
+      NUM10 = /#{PREFIX10}?#{COMPLEX10}/  # :nodoc:
+      NUM16 = /#{PREFIX16}#{COMPLEX16}/  # :nodoc:
+      NUM8 = /#{PREFIX8}#{COMPLEX8}/  # :nodoc:
+      NUM2 = /#{PREFIX2}#{COMPLEX2}/  # :nodoc:
+      NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/  # :nodoc:
+      
+    protected
+      
+      def scan_tokens encoder, options
+        
+        state = :initial
+        ident_kind = IDENT_KIND
+        
+        until eos?
+          
+          case state
+          when :initial
+            if match = scan(/ \s+ | \\\n /x)
+              encoder.text_token match, :space
+            elsif match = scan(/['\(\[\)\]]|#\(/)
+              encoder.text_token match, :operator  # FIXME: was :operator_fat
+            elsif match = scan(/;.*/)
+              encoder.text_token match, :comment
+            elsif match = scan(/#\\(?:newline|space|.?)/)
+              encoder.text_token match, :char
+            elsif match = scan(/#[ft]/)
+              encoder.text_token match, :pre_constant
+            elsif match = scan(/#{IDENTIFIER}/o)
+              encoder.text_token match, ident_kind[matched]
+            elsif match = scan(/\./)
+              encoder.text_token match, :operator
+            elsif match = scan(/"/)
+              encoder.begin_group :string
+              encoder.text_token match, :delimiter
+              state = :string
+            elsif match = scan(/#{NUM}/o) and not matched.empty?
+              encoder.text_token match, :integer
+            else
+              encoder.text_token getch, :error
+            end
+            
+          when :string
+            if match = scan(/[^"\\]+|\\.?/)
+              encoder.text_token match, :content
+            elsif match = scan(/"/)
+              encoder.text_token match, :delimiter
+              encoder.end_group :string
+              state = :initial
+            else
+              raise_inspect "else case \" reached; %p not handled." % peek(1),
+                encoder, state
+            end
+            
+          else
+            raise 'else case reached'
+            
+          end
+          
+        end
+        
+        if state == :string
+          encoder.end_group state
+        end
+        
+        encoder
+        
+      end
+    end
+  end
+end

data/lib/coderay/scanners/sql.rb

@@ -5,27 +5,26 @@ module CodeRay module Scanners
 
     register_for :sql
     
-    KEYWORDS = %w(
-      all and any as before begin between by case check collate
-      each else end exists
-      for foreign from full group having if in inner is join
-      like not of on or order outer over references
-      then to union using values when where
+    RESERVED_WORDS = %w(
+      all and as before begin by case collate
+      constraint create else end engine exists
+      for foreign from group if inner is join key
+      like not on or order outer primary references replace
+      then to trigger union using values when where
       left right distinct
     )
     
     OBJECTS = %w(
-      database databases table tables column columns fields index constraint
-      constraints transaction function procedure row key view trigger
+      database databases table tables column columns index
     )
     
     COMMANDS = %w(
       add alter comment create delete drop grant insert into select update set
-      show prompt begin commit rollback replace truncate
+      show
     )
     
     PREDEFINED_TYPES = %w(
-      char varchar varchar2 enum binary text tinytext mediumtext
+      char varchar enum binary text tinytext mediumtext
       longtext blob tinyblob mediumblob longblob timestamp
       date time datetime year double decimal float int
       integer tinyint mediumint bigint smallint unsigned bit
@@ -34,20 +33,16 @@ module CodeRay module Scanners
     
     PREDEFINED_FUNCTIONS = %w( sum cast substring abs pi count min max avg now )
     
-    DIRECTIVES = %w( 
-      auto_increment unique default charset initially deferred
-      deferrable cascade immediate read write asc desc after
-      primary foreign return engine
-    )
+    DIRECTIVES = %w( auto_increment unique default charset )
     
     PREDEFINED_CONSTANTS = %w( null true false )
     
-    IDENT_KIND = WordList::CaseIgnoring.new(:ident).
-      add(KEYWORDS, :keyword).
+    IDENT_KIND = CaseIgnoringWordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
       add(OBJECTS, :type).
       add(COMMANDS, :class).
-      add(PREDEFINED_TYPES, :predefined_type).
-      add(PREDEFINED_CONSTANTS, :predefined_constant).
+      add(PREDEFINED_TYPES, :pre_type).
+      add(PREDEFINED_CONSTANTS, :pre_constant).
       add(PREDEFINED_FUNCTIONS, :predefined).
       add(DIRECTIVES, :directive)
     
@@ -61,7 +56,6 @@ module CodeRay module Scanners
       state = :initial
       string_type = nil
       string_content = ''
-      name_expected = false
       
       until eos?
         
@@ -70,14 +64,13 @@ module CodeRay module Scanners
           if match = scan(/ \s+ | \\\n /x)
             encoder.text_token match, :space
           
-          elsif match = scan(/(?:--\s?|#).*/)
+          elsif match = scan(/^(?:--\s?|#).*/)
             encoder.text_token match, :comment
             
           elsif match = scan(%r( /\* (!)? (?: .*? \*/ | .* ) )mx)
             encoder.text_token match, self[1] ? :directive : :comment
             
-          elsif match = scan(/ [*\/=<>:;,!&^|()\[\]{}~%] | [-+\.](?!\d) /x)
-            name_expected = true if match == '.' && check(/[A-Za-z_]/)
+          elsif match = scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
             encoder.text_token match, :operator
             
           elsif match = scan(/(#{STRING_PREFIXES})?([`"'])/o)
@@ -90,23 +83,23 @@ module CodeRay module Scanners
             encoder.text_token match, :delimiter
             
           elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
-            encoder.text_token match, name_expected ? :ident : (match[0] == ?@ ? :variable : IDENT_KIND[match])
-            name_expected = false
+            # FIXME: Don't match keywords after "."
+            encoder.text_token match, match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
             
           elsif match = scan(/0[xX][0-9A-Fa-f]+/)
             encoder.text_token match, :hex
             
           elsif match = scan(/0[0-7]+(?![89.eEfF])/)
-            encoder.text_token match, :octal
+            encoder.text_token match, :oct
             
-          elsif match = scan(/[-+]?(?>\d+)(?![.eEfF])/)
+          elsif match = scan(/(?>\d+)(?![.eEfF])/)
             encoder.text_token match, :integer
             
-          elsif match = scan(/[-+]?(?:\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+)/)
+          elsif match = scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
             encoder.text_token match, :float
           
           elsif match = scan(/\\N/)
-            encoder.text_token match, :predefined_constant
+            encoder.text_token match, :pre_constant
             
           else
             encoder.text_token getch, :error

data/lib/coderay/scanners/yaml.rb

@@ -15,8 +15,9 @@ module Scanners
     
     def scan_tokens encoder, options
       
+      value_expected = nil
       state = :initial
-      key_indent = string_indent = 0
+      key_indent = indent = 0
       
       until eos?
         
@@ -54,16 +55,14 @@ module Scanners
           when match = scan(/[|>][-+]?/)
             encoder.begin_group :string
             encoder.text_token match, :delimiter
-            string_indent = key_indent || column(pos - match.size) - 1
+            string_indent = key_indent || column(pos - match.size - 1)
             encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
             encoder.end_group :string
             next
           when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
-            encoder.begin_group :string
-            encoder.text_token match, :content
-            string_indent = key_indent || column(pos - match.size) - 1
-            encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
-            encoder.end_group :string
+            encoder.text_token match, :string
+            string_indent = key_indent || column(pos - match.size - 1)
+            encoder.text_token matched, :string if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
             next
           end
           
@@ -78,7 +77,7 @@ module Scanners
             next
           when state == :initial && match = scan(/[\w.() ]*\S(?= *:(?: |$))/)
             encoder.text_token match, :key
-            key_indent = column(pos - match.size) - 1
+            key_indent = column(pos - match.size - 1)
             state = :colon
             next
           when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
@@ -87,7 +86,7 @@ module Scanners
             encoder.text_token match[1..-2], :content
             encoder.text_token match[-1,1], :delimiter
             encoder.end_group :key
-            key_indent = column(pos - match.size) - 1
+            key_indent = column(pos - match.size - 1)
             state = :colon
             next
           when match = scan(/(![\w\/]+)(:([\w:]+))?/)
@@ -107,10 +106,10 @@ module Scanners
             encoder.text_token match, :class_variable
             next
           when match = scan(/\d\d:\d\d:\d\d/)
-            encoder.text_token match, :octal
+            encoder.text_token match, :oct
             next
           when match = scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
-            encoder.text_token match, :octal
+            encoder.text_token match, :oct
             next
           when match = scan(/:\w+/)
             encoder.text_token match, :symbol