coderay 0.8.357 → 0.9.1

data/lib/coderay/scanners/scheme.rb

@@ -5,6 +5,9 @@ module CodeRay
     # Thanks to murphy for putting CodeRay into public.
     class Scheme < Scanner
       
+      # TODO: function defs
+      # TODO: built-in functions
+      
       register_for :scheme
       file_extension 'scm'
 

data/lib/coderay/scanners/sql.rb

@@ -0,0 +1,162 @@
+module CodeRay module Scanners
+  
+  # by Josh Goebel
+  class SQL < Scanner
+
+    register_for :sql
+    
+    RESERVED_WORDS = %w(
+      create database table index trigger drop primary key set select
+      insert update delete replace into
+      on from values before and or if exists case when
+      then else as group order by avg where
+      join inner outer union engine not
+      like end using collate show columns begin
+    )
+    
+    PREDEFINED_TYPES = %w(
+      char varchar enum binary text tinytext mediumtext
+      longtext blob tinyblob mediumblob longblob timestamp
+      date time datetime year double decimal float int
+      integer tinyint mediumint bigint smallint unsigned bit
+      bool boolean hex bin oct
+    )
+    
+    PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
+    
+    DIRECTIVES = %w( auto_increment unique default charset )
+
+    PREDEFINED_CONSTANTS = %w( null true false )
+    
+    IDENT_KIND = CaseIgnoringWordList.new(:ident).
+      add(RESERVED_WORDS, :reserved).
+      add(PREDEFINED_TYPES, :pre_type).
+      add(PREDEFINED_CONSTANTS, :pre_constant).
+      add(PREDEFINED_FUNCTIONS, :predefined).
+      add(DIRECTIVES, :directive)
+    
+    ESCAPE = / [rbfntv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
+    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+    
+    STRING_PREFIXES = /[xnb]|_\w+/i
+    
+    def scan_tokens tokens, options
+      
+      state = :initial
+      string_type = nil
+      string_content = ''
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        if state == :initial
+          
+          if scan(/ \s+ | \\\n /x)
+            kind = :space
+          
+          elsif scan(/^(?:--\s?|#).*/)
+            kind = :comment
+            
+          elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
+            kind = :comment
+            
+          elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
+            kind = :operator
+            
+          elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
+            prefix = self[1]
+            string_type = self[2]
+            tokens << [:open, :string]
+            tokens << [prefix, :modifier] if prefix
+            match = string_type
+            state = :string
+            kind = :delimiter
+            
+          elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
+            kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
+            
+          elsif scan(/0[xX][0-9A-Fa-f]+/)
+            kind = :hex
+            
+          elsif scan(/0[0-7]+(?![89.eEfF])/)
+            kind = :oct
+            
+          elsif scan(/(?>\d+)(?![.eEfF])/)
+            kind = :integer
+            
+          elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
+            kind = :float
+            
+          else
+            getch
+            kind = :error
+            
+          end
+          
+        elsif state == :string
+          if match = scan(/[^\\"'`]+/)
+            string_content << match
+            next
+          elsif match = scan(/["'`]/)
+            if string_type == match
+              if peek(1) == string_type  # doubling means escape
+                string_content << string_type << getch
+                next
+              end
+              unless string_content.empty?
+                tokens << [string_content, :content]
+                string_content = ''
+              end
+              tokens << [matched, :delimiter]
+              tokens << [:close, :string]
+              state = :initial
+              string_type = nil
+              next
+            else
+              string_content << match
+            end
+            next
+          elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+            unless string_content.empty?
+              tokens << [string_content, :content]
+              string_content = ''
+            end
+            kind = :char
+          elsif match = scan(/ \\ . /mox)
+            string_content << match
+            next
+          elsif scan(/ \\ | $ /x)
+            unless string_content.empty?
+              tokens << [string_content, :content]
+              string_content = ''
+            end
+            kind = :error
+            state = :initial
+          else
+            raise "else case \" reached; %p not handled." % peek(1), tokens
+          end
+          
+        else
+          raise 'else-case reached', tokens
+          
+        end
+        
+        match ||= matched
+        unless kind
+          raise_inspect 'Error token %p in line %d' %
+            [[match, kind], line], tokens, state
+        end
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+        
+      end
+      tokens
+      
+    end
+    
+  end
+  
+end end

data/lib/coderay/scanners/xml.rb

@@ -10,7 +10,7 @@ module Scanners
 
     register_for :xml
     file_extension 'xml'
-
+    
   end
 
 end

data/lib/coderay/scanners/yaml.rb

@@ -9,6 +9,8 @@ module Scanners
     register_for :yaml
     file_extension 'yml'
     
+    KINDS_NOT_LOC = :all
+    
     def scan_tokens tokens, options
       
       value_expected = nil
@@ -60,7 +62,8 @@ module Scanners
           when match = scan(/[|>][-+]?/)
             tokens << [:open, :string]
             tokens << [match, :delimiter]
-            tokens << [matched, :content] if scan(/(?:\n+ {#{key_indent + 1}}.*)+/)
+            string_indent = key_indent || column(pos - match.size - 1)
+            tokens << [matched, :content] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
             tokens << [:close, :string]
             next
           when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)

data/lib/coderay/styles/cycnus.rb

@@ -32,6 +32,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top }
   text-align: right;
 }
 .CodeRay .line_numbers tt { font-weight: bold }
+.CodeRay .line_numbers .highlighted { color: red }
 .CodeRay .no { padding: 0px 4px }
 .CodeRay .code { width: 100% }
 
@@ -57,9 +58,11 @@ ol.CodeRay li { white-space: pre }
 .ch .dl { color:#039 }
 
 .cl { color:#B06; font-weight:bold }
+.cm { color:#A08; font-weight:bold }
 .co { color:#036; font-weight:bold }
 .cr { color:#0A0 }
 .cv { color:#369 }
+.de { color:#B0B; }
 .df { color:#099; font-weight:bold }
 .di { color:#088; font-weight:bold }
 .dl { color:black }
@@ -69,7 +72,7 @@ ol.CodeRay li { white-space: pre }
 .e  { color:#666; font-weight:bold }
 .en { color:#800; font-weight:bold }
 .er { color:#F00; background-color:#FAA }
-.ex { color:#F00; font-weight:bold }
+.ex { color:#C00; font-weight:bold }
 .fl { color:#60E; font-weight:bold }
 .fu { color:#06B; font-weight:bold }
 .gv { color:#d70; font-weight:bold }
@@ -77,10 +80,11 @@ ol.CodeRay li { white-space: pre }
 .i  { color:#00D; font-weight:bold }
 .ic { color:#B44; font-weight:bold }
 
-.il { background: #eee; color: black }
-.il .il { background: #ddd }
-.il .il .il { background: #ccc }
-.il .idl { font-weight: bold; color: #777 }
+.il { background: #ddd; color: black }
+.il .il { background: #ccc }
+.il .il .il { background: #bbb }
+.il .idl { background: #ddd; font-weight: bold; color: #666 }
+.idl { background-color: #bbb; font-weight: bold; color: #666; }
 
 .im { color:#f00; }
 .in { color:#B2B; font-weight:bold }
@@ -93,8 +97,8 @@ ol.CodeRay li { white-space: pre }
 .pc { color:#038; font-weight:bold }
 .pd { color:#369; font-weight:bold }
 .pp { color:#579; }
-.ps { color:#00C; font-weight: bold; }
-.pt { color:#349; font-weight:bold }
+.ps { color:#00C; font-weight:bold }
+.pt { color:#074; font-weight:bold }
 .r, .kw  { color:#080; font-weight:bold }
 
 .ke { color: #808; }

data/lib/coderay/token_classes.rb

@@ -15,8 +15,10 @@ module CodeRay
       :class_variable => 'cv',
       :color => 'cr',
       :comment => 'c',
+      :complex => 'cm',
       :constant => 'co',
       :content => 'k',
+      :decorator => 'de',
       :definition => 'df',
       :delimiter => 'dl',
       :directive => 'di',
@@ -31,6 +33,7 @@ module CodeRay
       :function => 'fu',
       :global_variable => 'gv',
       :hex => 'hx',
+      :imaginary => 'cm',
       :important => 'im',
       :include => 'ic',
       :inline => 'il',
@@ -74,7 +77,7 @@ module CodeRay
       :space => :NO_HIGHLIGHT,  # 'sp'
       :plain => :NO_HIGHLIGHT,
     }
-    ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
+    ClassOfKind[:method] = ClassOfKind[:function]
     ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
     ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
     ClassOfKind[:escape] = ClassOfKind[:delimiter]

data/lib/coderay/tokens.rb

@@ -14,7 +14,7 @@ module CodeRay
   #
   #   [:comment, '# It looks like this']
   #   [:float, '3.1415926']
-  #   [:error, '���']
+  #   [:error, '$^']
   #
   # Some scanners also yield some kind of sub-tokens, represented by special
   # token texts, namely :open and :close .
@@ -46,47 +46,10 @@ module CodeRay
   #
   # Tokens' subclass TokenStream allows streaming to save memory.
   class Tokens < Array
-
-    class << self
-
-      # Convert the token to a string.
-      #
-      # This format is used by Encoders.Tokens.
-      # It can be reverted using read_token.
-      def write_token text, type
-        if text.is_a? String
-          "#{type}\t#{escape(text)}\n"
-        else
-          ":#{text}\t#{type}\t\n"
-        end
-      end
-
-      # Read a token from the string.
-      #
-      # Inversion of write_token.
-      #
-      # TODO Test this!
-      def read_token token
-        type, text = token.split("\t", 2)
-        if type[0] == ?:
-          [text.to_sym, type[1..-1].to_sym]
-        else
-          [type.to_sym, unescape(text)]
-        end
-      end
-
-      # Escapes a string for use in write_token.
-      def escape text
-        text.gsub(/[\n\\]/, '\\\\\&')
-      end
-
-      # Unescapes a string created by escape.
-      def unescape text
-        text.gsub(/\\[\n\\]/) { |m| m[1,1] }
-      end
-
-    end
-
+    
+    # The Scanner instance that created the tokens.
+    attr_accessor :scanner
+    
     # Whether the object is a TokenStream.
     #
     # Returns false.
@@ -146,7 +109,6 @@ module CodeRay
       encode :text, options
     end
 
-
     # Redirects unknown methods to encoder calls.
     #
     # For example, if you call +tokens.html+, the HTML encoder
@@ -230,6 +192,8 @@ module CodeRay
       replace fix
     end
     
+    # TODO: Scanner#split_into_lines
+    # 
     # Makes sure that:
     # - newlines are single tokens
     #   (which means all other token are single-line)
@@ -380,8 +344,48 @@ module CodeRay
 
   end
 
-  
-  # Token name abbreviations
-  require 'coderay/token_classes'
+end
 
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
 end
+
+__END__
+require 'test/unit'
+
+class TokensTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Tokens < Array
+    tokens = nil
+    assert_nothing_raised do
+      tokens = CodeRay::Tokens.new
+    end
+    assert_kind_of Array, tokens
+  end
+  
+  def test_adding_tokens
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens << ['string', :type]
+      tokens << ['()', :operator]
+    end
+    assert_equal tokens.size, 2
+  end
+  
+  def test_dump_undump
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens << ['string', :type]
+      tokens << ['()', :operator]
+    end
+    tokens2 = nil
+    assert_nothing_raised do
+      tokens2 = tokens.dump.undump
+    end
+    assert_equal tokens, tokens2
+  end
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: coderay
 version: !ruby/object:Gem::Version 
-  version: 0.8.357
+  version: 0.9.1
 platform: ruby
 authors: 
 - murphy
@@ -9,11 +9,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-02 00:00:00 +02:00
+date: 2010-01-01 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
-description: "  CodeRay is a Ruby library for syntax highlighting.\n  I try to make CodeRay easy to use and intuitive, but at the same time\n  fully featured, complete, fast and efficient.\n\n  Usage is simple:\n    require 'coderay'\n    code = 'some %q(weird (Ruby) can't shock) me!'\n    puts CodeRay.scan(code, :ruby).html\n"
+description: "  CodeRay is a Ruby library for syntax highlighting.\n  I try to make CodeRay easy to use and intuitive, but at the same time\n  fully featured, complete, fast and efficient.\n\n  Usage is simple:\n    CodeRay.scan(code, :ruby).div\n"
 email: murphy@rubychan.de
 executables: 
 - coderay
@@ -27,19 +27,24 @@ files:
 - ./lib/coderay/duo.rb
 - ./lib/coderay/encoder.rb
 - ./lib/coderay/encoders/_map.rb
+- ./lib/coderay/encoders/comment_filter.rb
 - ./lib/coderay/encoders/count.rb
 - ./lib/coderay/encoders/debug.rb
 - ./lib/coderay/encoders/div.rb
+- ./lib/coderay/encoders/filter.rb
 - ./lib/coderay/encoders/html/css.rb
 - ./lib/coderay/encoders/html/numerization.rb
 - ./lib/coderay/encoders/html/output.rb
 - ./lib/coderay/encoders/html.rb
+- ./lib/coderay/encoders/json.rb
+- ./lib/coderay/encoders/lines_of_code.rb
 - ./lib/coderay/encoders/null.rb
 - ./lib/coderay/encoders/page.rb
 - ./lib/coderay/encoders/span.rb
 - ./lib/coderay/encoders/statistic.rb
+- ./lib/coderay/encoders/term.rb
 - ./lib/coderay/encoders/text.rb
-- ./lib/coderay/encoders/tokens.rb
+- ./lib/coderay/encoders/token_class_filter.rb
 - ./lib/coderay/encoders/xml.rb
 - ./lib/coderay/encoders/yaml.rb
 - ./lib/coderay/for_redcloth.rb
@@ -50,21 +55,26 @@ files:
 - ./lib/coderay/scanner.rb
 - ./lib/coderay/scanners/_map.rb
 - ./lib/coderay/scanners/c.rb
+- ./lib/coderay/scanners/cpp.rb
 - ./lib/coderay/scanners/css.rb
 - ./lib/coderay/scanners/debug.rb
 - ./lib/coderay/scanners/delphi.rb
 - ./lib/coderay/scanners/diff.rb
+- ./lib/coderay/scanners/groovy.rb
 - ./lib/coderay/scanners/html.rb
 - ./lib/coderay/scanners/java/builtin_types.rb
 - ./lib/coderay/scanners/java.rb
 - ./lib/coderay/scanners/java_script.rb
 - ./lib/coderay/scanners/json.rb
 - ./lib/coderay/scanners/nitro_xhtml.rb
+- ./lib/coderay/scanners/php.rb
 - ./lib/coderay/scanners/plaintext.rb
+- ./lib/coderay/scanners/python.rb
 - ./lib/coderay/scanners/rhtml.rb
 - ./lib/coderay/scanners/ruby/patterns.rb
 - ./lib/coderay/scanners/ruby.rb
 - ./lib/coderay/scanners/scheme.rb
+- ./lib/coderay/scanners/sql.rb
 - ./lib/coderay/scanners/xml.rb
 - ./lib/coderay/scanners/yaml.rb
 - ./lib/coderay/style.rb