coderay 0.7.4.215 → 0.8.260

data/lib/coderay/scanners/json.rb

@@ -0,0 +1,106 @@
+module CodeRay
+module Scanners
+  
+  class JSON < Scanner
+    
+    include Streamable
+    
+    register_for :json
+    
+    CONSTANTS = %w( true false null )
+    IDENT_KIND = WordList.new(:key).add(CONSTANTS, :reserved)
+    
+    ESCAPE = / [bfnrt\\"\/] /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      stack = []
+      string_delimiter = nil
+      key_expected = false
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        case state
+        
+        when :initial
+          if match = scan(/ \s+ | \\\n /x)
+            tokens << [match, :space]
+            next
+          elsif match = scan(/ [:,\[{\]}] /x)
+            kind = :operator
+            case match
+            when '{': stack << :object; key_expected = true
+            when '[': stack << :array
+            when ':': key_expected = false
+            when ',': key_expected = true if stack.last == :object
+            when '}', ']': stack.pop  # no error recovery, but works for valid JSON
+            end
+          elsif match = scan(/ true | false | null /x)
+            kind = IDENT_KIND[match]
+          elsif match = scan(/-?(?:0|[1-9]\d*)/)
+            kind = :integer
+            if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
+              match << matched
+              kind = :float
+            end
+          elsif match = scan(/"/)
+            state = key_expected ? :key : :string
+            tokens << [:open, state]
+            kind = :delimiter
+          else
+            getch
+            kind = :error
+          end
+          
+        when :string, :key
+          if scan(/[^\\"]+/)
+            kind = :content
+          elsif scan(/"/)
+            tokens << ['"', :delimiter]
+            tokens << [:close, state]
+            state = :initial
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          elsif scan(/\\./m)
+            kind = :content
+          elsif scan(/ \\ | $ /x)
+            tokens << [:close, :delimiter]
+            kind = :error
+            state = :initial
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+          
+        else
+          raise_inspect 'Unknown state', tokens
+          
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+        
+      end
+      
+      if [:string, :key].include? state
+        tokens << [:close, state]
+      end
+      
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/nitro_xhtml.rb

@@ -11,6 +11,7 @@ module Scanners
 
     include Streamable
     register_for :nitro_xhtml
+    file_extension :xhtml
 
     NITRO_RUBY_BLOCK = /
       <\?r
@@ -95,20 +96,20 @@ module Scanners
           delimiter = CLOSING_PAREN[start_tag[1,1]]
           end_tag = match[-1,1] == delimiter ? delimiter : ''
           tokens << [:open, :inline]
-          tokens << [start_tag, :delimiter]
+          tokens << [start_tag, :inline_delimiter]
           code = match[start_tag.size .. -1 - end_tag.size]
           @ruby_scanner.tokenize code
-          tokens << [end_tag, :delimiter] unless end_tag.empty?
+          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
           tokens << [:close, :inline]
 
         elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
           start_tag = '<?r'
           end_tag = match[-2,2] == '?>' ? '?>' : ''
           tokens << [:open, :inline]
-          tokens << [start_tag, :delimiter]
+          tokens << [start_tag, :inline_delimiter]
           code = match[start_tag.size .. -(end_tag.size)-1]
           @ruby_scanner.tokenize code
-          tokens << [end_tag, :delimiter] unless end_tag.empty?
+          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
           tokens << [:close, :inline]
 
         elsif entity = scan(/#{NITRO_ENTITY}/o)

data/lib/coderay/scanners/plaintext.rb

@@ -4,6 +4,8 @@ module Scanners
   class Plaintext < Scanner
 
     register_for :plaintext, :plain
+    
+    include Streamable
 
     def scan_tokens tokens, options
       text = (scan_until(/\z/) || '')

data/lib/coderay/scanners/rhtml.rb

@@ -51,10 +51,10 @@ module Scanners
           start_tag = match[/\A<%[-=]?/]
           end_tag = match[/-?%?>?\z/]
           tokens << [:open, :inline]
-          tokens << [start_tag, :delimiter]
+          tokens << [start_tag, :inline_delimiter]
           code = match[start_tag.size .. -1 - end_tag.size]
           @ruby_scanner.tokenize code
-          tokens << [end_tag, :delimiter] unless end_tag.empty?
+          tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
           tokens << [:close, :inline]
 
         else

data/lib/coderay/scanners/ruby.rb

@@ -18,6 +18,7 @@ module Scanners
     include Streamable
 
     register_for :ruby
+    file_extension 'rb'
 
     helper :patterns
 
@@ -90,15 +91,15 @@ module Scanners
             end
 
           when '#'
-            case peek(1)[0]
-            when ?{
+            case peek(1)
+            when '{'
               inline_block_stack << [state, depth, heredocs]
               value_expected = true
               state = :initial
               depth = 1
               tokens << [:open, :inline]
               tokens << [match + getch, :inline_delimiter]
-            when ?$, ?@
+            when '$', '@'
               tokens << [match, :escape]
               last_state = state  # scan one token as normal code, then return here
               state = :initial
@@ -121,36 +122,37 @@ module Scanners
 # }}}
         else
 # {{{
-          if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
-            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
-            case m = match[0]
-            when ?\s, ?\t, ?\f
-              match << scan(/\s*/) unless eos? or heredocs
-              kind = :space
-            when ?\n, ?\\
-              kind = :space
-              if m == ?\n
-                value_expected = true  # FIXME not quite true
-                state = :initial if state == :undef_comma_expected
-              end
-              if heredocs
-                unscan  # heredoc scanning needs \n at start
-                state = heredocs.shift
-                tokens << [:open, state.type]
-                heredocs = nil if heredocs.empty?
-                next
-              else
-                match << scan(/\s*/) unless eos?
-              end
-            when ?#, ?=, ?_
-              kind = :comment
-              value_expected = true
+          if match = scan(/[ \t\f]+/)
+            kind = :space
+            match << scan(/\s*/) unless eos? || heredocs
+            value_expected = true if match.index(?\n) # FIXME not quite true
+            tokens << [match, kind]
+            next
+            
+          elsif match = scan(/\\?\n/)
+            kind = :space
+            if match == "\n"
+              value_expected = true  # FIXME not quite true
+              state = :initial if state == :undef_comma_expected
+            end
+            if heredocs
+              unscan  # heredoc scanning needs \n at start
+              state = heredocs.shift
+              tokens << [:open, state.type]
+              heredocs = nil if heredocs.empty?
+              next
             else
-              raise_inspect 'else-case _ reached, because case %p was
-                not handled' % [matched[0].chr], tokens
+              match << scan(/\s*/) unless eos?
             end
             tokens << [match, kind]
             next
+          
+          elsif match = scan(/\#.*/) or
+            ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
+              kind = :comment
+              value_expected = true
+              tokens << [match, kind]
+              next
 
           elsif state == :initial
 
@@ -167,19 +169,19 @@ module Scanners
                 end
               end
               ## experimental!
-              value_expected = :set if
-                patterns::REGEXP_ALLOWED[match] or check(/#{patterns::VALUE_FOLLOWS}/o)
+              value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
             
             elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o)
               kind = :ident
               value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
 
             # OPERATORS #
-            elsif not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)
+            # TODO: match (), [], {} as one single operator
+            elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
               if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
                 value_expected = :set
               end
-              last_token_dot = :set if match == '.' or match == '::'
+              last_token_dot = :set if self[1]
               kind = :operator
               unless inline_block_stack.empty?
                 case match
@@ -210,8 +212,9 @@ module Scanners
               interpreted = true
               state = patterns::StringState.new :regexp, interpreted, match
 
-            elsif match = scan(/#{patterns::NUMERIC}/o)
-              kind = if self[1] then :float else :integer end
+            # elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
+            elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
+              kind = self[1] ? :float : :integer
 
             elsif match = scan(/#{patterns::SYMBOL}/o)
               case delim = match[1]
@@ -285,6 +288,18 @@ module Scanners
               next
             end
 
+          elsif state == :module_expected
+            if match = scan(/<</)
+              kind = :operator
+            else
+              state = :initial
+              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
+                kind = :class
+              else
+                next
+              end
+            end
+
           elsif state == :undef_expected
             state = :undef_comma_expected
             if match = scan(/#{patterns::METHOD_NAME_EX}/o)
@@ -306,6 +321,15 @@ module Scanners
               next
             end
 
+          elsif state == :alias_expected
+            if match = scan(/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
+              tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
+              tokens << [self[2], :space]
+              tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
+            end
+            state = :initial
+            next
+
           elsif state == :undef_comma_expected
             if match = scan(/,/)
               kind = :operator
@@ -315,23 +339,13 @@ module Scanners
               next
             end
 
-          elsif state == :module_expected
-            if match = scan(/<</)
-              kind = :operator
-            else
-              state = :initial
-              if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
-                kind = :class
-              else
-                next
-              end
-            end
-
           end
 # }}}
-
-          value_expected = value_expected == :set
-          last_token_dot = last_token_dot == :set
+          
+          unless kind == :error
+            value_expected = value_expected == :set
+            last_token_dot = last_token_dot == :set
+          end
 
           if $DEBUG and not kind
             raise_inspect 'Error token %p in line %d' %

data/lib/coderay/scanners/ruby/patterns.rb

@@ -14,19 +14,14 @@ module Scanners
 
     DEF_KEYWORDS = %w[ def ]
     UNDEF_KEYWORDS = %w[ undef ]
+    ALIAS_KEYWORDS = %w[ alias ]
     MODULE_KEYWORDS = %w[class module]
     DEF_NEW_STATE = WordList.new(:initial).
       add(DEF_KEYWORDS, :def_expected).
       add(UNDEF_KEYWORDS, :undef_expected).
+      add(ALIAS_KEYWORDS, :alias_expected).
       add(MODULE_KEYWORDS, :module_expected)
 
-    IDENTS_ALLOWING_REGEXP = %w[
-      and or not while until unless if then elsif when sub sub! gsub gsub!
-      scan slice slice! split
-    ]
-    REGEXP_ALLOWED = WordList.new(false).
-      add(IDENTS_ALLOWING_REGEXP, :set)
-
     PREDEFINED_CONSTANTS = %w[
       nil true false self
       DATA ARGV ARGF __FILE__ __LINE__
@@ -41,19 +36,20 @@ module Scanners
     METHOD_NAME = / #{IDENT} [?!]? /ox
     METHOD_NAME_OPERATOR = /
       \*\*?           # multiplication and power
-      | [-+]@?        # plus, minus
-      | [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
+      | [-+~]@?       # plus, minus, tilde with and without @
+      | [\/%&|^`]     # division, modulo or format strings, &and, |or, ^xor, `system`
       | \[\]=?        # array getter and setter
       | << | >>       # append or shift left, shift right
       | <=?>? | >=?   # comparison, rocket operator
-      | ===?          # simple equality and case equality
+      | ===? | =~     # simple equality, case equality, match
+      | ![~=@]?       # negation with and without @, not-equal and not-match
     /ox
     METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
     INSTANCE_VARIABLE = / @ #{IDENT} /ox
     CLASS_VARIABLE = / @@ #{IDENT} /ox
     OBJECT_VARIABLE = / @@? #{IDENT} /ox
     GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
-    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
+    PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
     VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
 
     QUOTE_TO_TYPE = {
@@ -73,7 +69,7 @@ module Scanners
     EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
     FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
     FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
-    NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
+    NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
 
     SYMBOL = /
       :
@@ -83,6 +79,7 @@ module Scanners
       | ['"]
       )
     /ox
+    METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
 
     # TODO investigste \M, \c and \C escape sequences
     # (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
@@ -111,7 +108,7 @@ module Scanners
       (?:
         ( [A-Za-z_0-9]+ )  # $2 = delim
       |
-        ( ["'`] )          # $3 = quote, type
+        ( ["'`\/] )        # $3 = quote, type
         ( [^\n]*? ) \3     # $4 = delim
       )
     /mx
@@ -129,15 +126,14 @@ module Scanners
     /mx
     
     # Checks for a valid value to follow. This enables
-    # fancy_allowed in method calls.
+    # value_expected in method calls without parentheses.
     VALUE_FOLLOWS = /
-      \s+
+      (?>[ \t\f\v]+)
       (?:
         [%\/][^\s=]
-      |
-        <<-?\S
-      |
-        #{CHARACTER}
+      | <<-?\S
+      | [-+] \d
+      | #{CHARACTER}
       )
     /x
 

data/lib/coderay/scanners/scheme.rb

@@ -0,0 +1,142 @@
+module CodeRay
+  module Scanners
+
+    # Scheme scanner for CodeRay (by closure).
+    # Thanks to murphy for putting CodeRay into public.
+    class Scheme < Scanner
+      
+      register_for :scheme
+      file_extension 'scm'
+
+      CORE_FORMS = %w[
+        lambda let let* letrec syntax-case define-syntax let-syntax
+        letrec-syntax begin define quote if or and cond case do delay
+        quasiquote set! cons force call-with-current-continuation call/cc
+      ]
+
+      IDENT_KIND = CaseIgnoringWordList.new(:ident).
+        add(CORE_FORMS, :reserved)
+      
+      #IDENTIFIER_INITIAL = /[a-z!@\$%&\*\/\:<=>\?~_\^]/i
+      #IDENTIFIER_SUBSEQUENT = /#{IDENTIFIER_INITIAL}|\d|\.|\+|-/
+      #IDENTIFIER = /#{IDENTIFIER_INITIAL}#{IDENTIFIER_SUBSEQUENT}*|\+|-|\.{3}/
+      IDENTIFIER = /[a-zA-Z!@$%&*\/:<=>?~_^][\w!@$%&*\/:<=>?~^.+\-]*|[+-]|\.\.\./
+      DIGIT = /\d/
+      DIGIT10 = DIGIT
+      DIGIT16 = /[0-9a-f]/i
+      DIGIT8 = /[0-7]/
+      DIGIT2 = /[01]/
+      RADIX16 = /\#x/i
+      RADIX8 = /\#o/i
+      RADIX2 = /\#b/i
+      RADIX10 = /\#d/i
+      EXACTNESS = /#i|#e/i
+      SIGN = /[\+-]?/
+      EXP_MARK = /[esfdl]/i
+      EXP = /#{EXP_MARK}#{SIGN}#{DIGIT}+/
+      SUFFIX = /#{EXP}?/
+      PREFIX10 = /#{RADIX10}?#{EXACTNESS}?|#{EXACTNESS}?#{RADIX10}?/
+      PREFIX16 = /#{RADIX16}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX16}/
+      PREFIX8 = /#{RADIX8}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX8}/
+      PREFIX2 = /#{RADIX2}#{EXACTNESS}?|#{EXACTNESS}?#{RADIX2}/
+      UINT10 = /#{DIGIT10}+#*/
+      UINT16 = /#{DIGIT16}+#*/
+      UINT8 = /#{DIGIT8}+#*/
+      UINT2 = /#{DIGIT2}+#*/
+      DECIMAL = /#{DIGIT10}+#+\.#*#{SUFFIX}|#{DIGIT10}+\.#{DIGIT10}*#*#{SUFFIX}|\.#{DIGIT10}+#*#{SUFFIX}|#{UINT10}#{EXP}/
+      UREAL10 = /#{UINT10}\/#{UINT10}|#{DECIMAL}|#{UINT10}/
+      UREAL16 = /#{UINT16}\/#{UINT16}|#{UINT16}/
+      UREAL8 = /#{UINT8}\/#{UINT8}|#{UINT8}/
+      UREAL2 = /#{UINT2}\/#{UINT2}|#{UINT2}/
+      REAL10 = /#{SIGN}#{UREAL10}/
+      REAL16 = /#{SIGN}#{UREAL16}/
+      REAL8 = /#{SIGN}#{UREAL8}/
+      REAL2 = /#{SIGN}#{UREAL2}/
+      IMAG10 = /i|#{UREAL10}i/
+      IMAG16 = /i|#{UREAL16}i/
+      IMAG8 = /i|#{UREAL8}i/
+      IMAG2 = /i|#{UREAL2}i/
+      COMPLEX10 = /#{REAL10}@#{REAL10}|#{REAL10}\+#{IMAG10}|#{REAL10}-#{IMAG10}|\+#{IMAG10}|-#{IMAG10}|#{REAL10}/
+      COMPLEX16 = /#{REAL16}@#{REAL16}|#{REAL16}\+#{IMAG16}|#{REAL16}-#{IMAG16}|\+#{IMAG16}|-#{IMAG16}|#{REAL16}/
+      COMPLEX8 = /#{REAL8}@#{REAL8}|#{REAL8}\+#{IMAG8}|#{REAL8}-#{IMAG8}|\+#{IMAG8}|-#{IMAG8}|#{REAL8}/
+      COMPLEX2 = /#{REAL2}@#{REAL2}|#{REAL2}\+#{IMAG2}|#{REAL2}-#{IMAG2}|\+#{IMAG2}|-#{IMAG2}|#{REAL2}/
+      NUM10 = /#{PREFIX10}?#{COMPLEX10}/
+      NUM16 = /#{PREFIX16}#{COMPLEX16}/
+      NUM8 = /#{PREFIX8}#{COMPLEX8}/
+      NUM2 = /#{PREFIX2}#{COMPLEX2}/
+      NUM = /#{NUM10}|#{NUM16}|#{NUM8}|#{NUM2}/
+    
+    private
+      def scan_tokens tokens,options
+        
+        state = :initial
+        ident_kind = IDENT_KIND
+        
+        until eos?
+          kind = match = nil
+          
+          case state
+          when :initial
+            if scan(/ \s+ | \\\n /x)
+              kind = :space
+            elsif scan(/['\(\[\)\]]|#\(/)
+              kind = :operator_fat
+            elsif scan(/;.*/)
+              kind = :comment
+            elsif scan(/#\\(?:newline|space|.?)/)
+              kind = :char
+            elsif scan(/#[ft]/)
+              kind = :pre_constant
+            elsif scan(/#{IDENTIFIER}/o)
+              kind = ident_kind[matched]
+            elsif scan(/\./)
+              kind = :operator
+            elsif scan(/"/)
+              tokens << [:open, :string]
+              state = :string
+              tokens << ['"', :delimiter]
+              next
+            elsif scan(/#{NUM}/o) and not matched.empty?
+              kind = :integer
+            elsif getch
+              kind = :error
+            end
+            
+          when :string
+            if scan(/[^"\\]+/) or scan(/\\.?/)
+              kind = :content
+            elsif scan(/"/)
+              tokens << ['"', :delimiter]
+              tokens << [:close, :string]
+              state = :initial
+              next
+            else
+              raise_inspect "else case \" reached; %p not handled." % peek(1),
+                tokens, state
+            end
+            
+          else
+            raise "else case reached"
+          end
+          
+          match ||= matched
+          if $DEBUG and not kind
+            raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+          end
+          raise_inspect 'Empty token', tokens, state unless match
+          
+          tokens << [match, kind]
+          
+        end  # until eos
+        
+        if state == :string
+          tokens << [:close, :string]
+        end
+        
+        tokens
+        
+      end #scan_tokens
+    end #class
+  end #module scanners
+end #module coderay