coderay-beta 0.9.1

data/lib/coderay/scanners/delphi.rb

@@ -0,0 +1,150 @@
+module CodeRay
+module Scanners
+  
+  class Delphi < Scanner
+
+    register_for :delphi
+    file_extension 'pas'
+    
+    RESERVED_WORDS = [
+      'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
+      'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
+      'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
+      'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in',
+      'inherited', 'initialization', 'inline', 'interface', 'is', 'label',
+      'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed',
+      'procedure', 'program', 'property', 'raise', 'record', 'repeat',
+      'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
+      'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
+      'xor', 'on'
+    ]
+
+    DIRECTIVES = [
+      'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
+      'contains', 'deprecated', 'dispid', 'dynamic', 'export',
+      'external', 'far', 'forward', 'implements', 'local', 
+      'near', 'nodefault', 'on', 'overload', 'override',
+      'package', 'pascal', 'platform', 'private', 'protected', 'public',
+      'published', 'read', 'readonly', 'register', 'reintroduce',
+      'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
+      'virtual', 'write', 'writeonly'
+    ]
+
+    IDENT_KIND = CaseIgnoringWordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(DIRECTIVES, :directive)
+    
+    NAME_FOLLOWS = CaseIgnoringWordList.new(false).
+      add(%w(procedure function .))
+
+  private
+    def scan_tokens tokens, options
+
+      state = :initial
+      last_token = ''
+
+      until eos?
+
+        kind = nil
+        match = nil
+
+        if state == :initial
+          
+          if scan(/ \s+ /x)
+            tokens << [matched, :space]
+            next
+            
+          elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
+            tokens << [matched, :preprocessor]
+            next
+            
+          elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
+            tokens << [matched, :comment]
+            next
+            
+          elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
+            kind = :operator
+          
+          elsif match = scan(/\./)
+            kind = :operator
+            if last_token == 'end'
+              tokens << [match, kind]
+              next
+            end
+            
+          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
+            
+          elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
+            tokens << [:open, :char]
+            tokens << ["'", :delimiter]
+            tokens << [self[1], :content]
+            tokens << ["'", :delimiter]
+            tokens << [:close, :char]
+            next
+            
+          elsif match = scan(/ ' /x)
+            tokens << [:open, :string]
+            state = :string
+            kind = :delimiter
+            
+          elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
+            kind = :char
+            
+          elsif scan(/ \$ [0-9A-Fa-f]+ /x)
+            kind = :hex
+            
+          elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
+            kind = :integer
+            
+          elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
+            kind = :float
+
+          else
+            kind = :error
+            getch
+
+          end
+          
+        elsif state == :string
+          if scan(/[^\n']+/)
+            kind = :content
+          elsif scan(/''/)
+            kind = :char
+          elsif scan(/'/)
+            tokens << ["'", :delimiter]
+            tokens << [:close, :string]
+            state = :initial
+            next
+          elsif scan(/\n/)
+            tokens << [:close, :string]
+            kind = :error
+            state = :initial
+          else
+            raise "else case \' reached; %p not handled." % peek(1), tokens
+          end
+          
+        else
+          raise 'else-case reached', tokens
+          
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens unless match
+
+        last_token = match
+        tokens << [match, kind]
+        
+      end
+      
+      tokens
+    end
+
+  end
+
+end
+end

data/lib/coderay/scanners/diff.rb

@@ -0,0 +1,105 @@
+module CodeRay
+module Scanners
+  
+  class Diff < Scanner
+    
+    register_for :diff
+    title 'diff output'
+    
+    def scan_tokens tokens, options
+      
+      line_kind = nil
+      state = :initial
+      
+      until eos?
+        kind = match = nil
+        
+        if match = scan(/\n/)
+          if line_kind
+            tokens << [:end_line, line_kind]
+            line_kind = nil
+          end
+          tokens << [match, :space]
+          next
+        end
+        
+        case state
+        
+        when :initial
+          if match = scan(/--- |\+\+\+ |=+|_+/)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/Index: |Property changes on: /)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/Added: /)
+            tokens << [:begin_line, line_kind = :head]
+            tokens << [match, :head]
+            next unless match = scan(/.+/)
+            kind = :plain
+            state = :added
+          elsif match = scan(/\\ /)
+            tokens << [:begin_line, line_kind = :change]
+            tokens << [match, :change]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif scan(/(@@)((?>[^@\n]*))(@@)/)
+            tokens << [:begin_line, line_kind = :change]
+            tokens << [self[1], :change]
+            tokens << [self[2], :plain]
+            tokens << [self[3], :change]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/\+/)
+            tokens << [:begin_line, line_kind = :insert]
+            tokens << [match, :insert]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif match = scan(/-/)
+            tokens << [:begin_line, line_kind = :delete]
+            tokens << [match, :delete]
+            next unless match = scan(/.+/)
+            kind = :plain
+          elsif scan(/ .*/)
+            kind = :comment
+          elsif scan(/.+/)
+            tokens << [:begin_line, line_kind = :head]
+            kind = :plain
+          else
+            raise_inspect 'else case rached'
+          end
+        
+        when :added
+          if match = scan(/   \+/)
+            tokens << [:begin_line, line_kind = :insert]
+            tokens << [match, :insert]
+            next unless match = scan(/.+/)
+            kind = :plain
+          else
+            state = :initial
+            next
+          end
+        end
+        
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+      end
+      
+      tokens << [:end_line, line_kind] if line_kind
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/scanners/groovy.rb

@@ -0,0 +1,263 @@
+module CodeRay
+module Scanners
+
+  load :java
+
+  class Groovy < Java
+
+    include Streamable
+    register_for :groovy
+    
+    # TODO: Check this!
+    GROOVY_KEYWORDS = %w[
+      as assert def in
+    ]
+    KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
+      case instanceof new return throw typeof while as assert in
+    ]
+    GROOVY_MAGIC_VARIABLES = %w[ it ]
+    
+    IDENT_KIND = Java::IDENT_KIND.dup.
+      add(GROOVY_KEYWORDS, :keyword).
+      add(GROOVY_MAGIC_VARIABLES, :local_variable)
+    
+    ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x  # no 4-byte unicode chars? U[a-fA-F0-9]{8}
+    REGEXP_ESCAPE =  / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
+    
+    # TODO: interpretation inside ', ", /
+    STRING_CONTENT_PATTERN = {
+      "'" => /(?>\\[^\\'\n]+|[^\\'\n]+)+/,
+      '"' => /[^\\$"\n]+/,
+      "'''" => /(?>[^\\']+|'(?!''))+/,
+      '"""' => /(?>[^\\$"]+|"(?!""))+/,
+      '/' => /[^\\$\/\n]+/,
+    }
+    
+    def scan_tokens tokens, options
+
+      state = :initial
+      inline_block_stack = []
+      inline_block_paren_depth = nil
+      string_delimiter = nil
+      import_clause = class_name_follows = last_token = after_def = false
+      value_expected = true
+
+      until eos?
+
+        kind = nil
+        match = nil
+        
+        case state
+
+        when :initial
+
+          if match = scan(/ \s+ | \\\n /x)
+            tokens << [match, :space]
+            if match.index ?\n
+              import_clause = after_def = false
+              value_expected = true unless value_expected
+            end
+            next
+          
+          elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+            value_expected = true
+            after_def = false
+            kind = :comment
+          
+          elsif bol? && scan(/ \#!.* /x)
+            kind = :doctype
+          
+          elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
+            after_def = value_expected = false
+            kind = :include
+          
+          elsif match = scan(/ #{IDENT} | \[\] /ox)
+            kind = IDENT_KIND[match]
+            value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
+            if last_token == '.'
+              kind = :ident
+            elsif class_name_follows
+              kind = :class
+              class_name_follows = false
+            elsif after_def && check(/\s*[({]/)
+              kind = :method
+              after_def = false
+            elsif kind == :ident && last_token != '?' && check(/:/)
+              kind = :key
+            else
+              class_name_follows = true if match == 'class' || (import_clause && match == 'as')
+              import_clause = match == 'import'
+              after_def = true if match == 'def'
+            end
+          
+          elsif scan(/;/)
+            import_clause = after_def = false
+            value_expected = true
+            kind = :operator
+          
+          elsif scan(/\{/)
+            class_name_follows = after_def = false
+            value_expected = true
+            kind = :operator
+            if !inline_block_stack.empty?
+              inline_block_paren_depth += 1
+            end
+          
+          # TODO: ~'...', ~"..." and ~/.../ style regexps
+          elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
+              && | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
+            value_expected = true
+            value_expected = :regexp if match == '~'
+            after_def = false
+            kind = :operator
+          
+          elsif match = scan(/ [)\]}] /x)
+            value_expected = after_def = false
+            if !inline_block_stack.empty? && match == '}'
+              inline_block_paren_depth -= 1
+              if inline_block_paren_depth == 0  # closing brace of inline block reached
+                tokens << [match, :inline_delimiter]
+                tokens << [:close, :inline]
+                state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
+                next
+              end
+            end
+          
+          elsif check(/[\d.]/)
+            after_def = value_expected = false
+            if scan(/0[xX][0-9A-Fa-f]+/)
+              kind = :hex
+            elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
+              kind = :oct
+            elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
+              kind = :float
+            elsif scan(/\d+[lLgG]?/)
+              kind = :integer
+            end
+
+          elsif match = scan(/'''|"""/)
+            after_def = value_expected = false
+            state = :multiline_string
+            tokens << [:open, :string]
+            string_delimiter = match
+            kind = :delimiter
+          
+          # TODO: record.'name'
+          elsif match = scan(/["']/)
+            after_def = value_expected = false
+            state = match == '/' ? :regexp : :string
+            tokens << [:open, state]
+            string_delimiter = match
+            kind = :delimiter
+
+          elsif value_expected && (match = scan(/\//))
+            after_def = value_expected = false
+            tokens << [:open, :regexp]
+            state = :regexp
+            string_delimiter = '/'
+            kind = :delimiter
+
+          elsif scan(/ @ #{IDENT} /ox)
+            after_def = value_expected = false
+            kind = :annotation
+
+          elsif scan(/\//)
+            after_def = false
+            value_expected = true
+            kind = :operator
+          
+          else
+            getch
+            kind = :error
+
+          end
+
+        when :string, :regexp, :multiline_string
+          if scan(STRING_CONTENT_PATTERN[string_delimiter])
+            kind = :content
+            
+          elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
+            tokens << [match, :delimiter]
+            if state == :regexp
+              # TODO: regexp modifiers? s, m, x, i?
+              modifiers = scan(/[ix]+/)
+              tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
+            end
+            state = :string if state == :multiline_string
+            tokens << [:close, state]
+            string_delimiter = nil
+            after_def = value_expected = false
+            state = :initial
+            next
+          
+          elsif (state == :string || state == :multiline_string) &&
+              (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
+            if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
+              kind = :content
+            else
+              kind = :char
+            end
+          elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            kind = :char
+          
+          elsif match = scan(/ \$ #{IDENT} /mox)
+            tokens << [:open, :inline]
+            tokens << ['$', :inline_delimiter]
+            match = match[1..-1]
+            tokens << [match, IDENT_KIND[match]]
+            tokens << [:close, :inline]
+            next
+          elsif match = scan(/ \$ \{ /x)
+            tokens << [:open, :inline]
+            tokens << ['${', :inline_delimiter]
+            inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
+            inline_block_paren_depth = 1
+            state = :initial
+            next
+          
+          elsif scan(/ \$ /mx)
+            kind = :content
+          
+          elsif scan(/ \\. /mx)
+            kind = :content
+          
+          elsif scan(/ \\ | \n /x)
+            tokens << [:close, state]
+            kind = :error
+            after_def = value_expected = false
+            state = :initial
+          
+          else
+            raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
+          end
+
+        else
+          raise_inspect 'Unknown state', tokens
+
+        end
+
+        match ||= matched
+        if $DEBUG and not kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        last_token = match unless [:space, :comment, :doctype].include? kind
+        
+        tokens << [match, kind]
+
+      end
+
+      if [:multiline_string, :string, :regexp].include? state
+        tokens << [:close, state]
+      end
+
+      tokens
+    end
+
+  end
+
+end
+end