coderay 0.8.260 → 0.8.263

data/lib/coderay/encoder.rb

@@ -133,7 +133,7 @@ module CodeRay
       # whether +text+ is a String.
       def token text, kind
         out =
-          if text.is_a? ::String  # Ruby 1.9: watch out, :open.is_a? String is true
+          if text.is_a? ::String
             text_token text, kind
           elsif text.is_a? ::Symbol
             block_token text, kind
@@ -171,9 +171,14 @@ module CodeRay
       #
       # The already created +tokens+ object must be used; it can be a
       # TokenStream or a Tokens object.
-      def compile tokens, options
-        tokens.each { |text, kind| token text, kind }  # FIXME for Ruby 1.9?
-        #tokens.each(&self)
+      if RUBY_VERSION >= '1.9'
+        def compile tokens, options
+          tokens.each { |text, kind| token text, kind }  # FIXME for Ruby 1.9?
+        end
+      else
+        def compile tokens, options
+          tokens.each(&self)
+        end
       end
 
     end

data/lib/coderay/encoders/html.rb

@@ -220,8 +220,13 @@ module Encoders
       super
     end
 
-    def token text, type
-      if text.is_a? ::String
+    def token text, type = :plain
+      case text
+      
+      when nil
+        # raise 'Token with nil as text was given: %p' % [[text, type]] 
+      
+      when String
         if text =~ /#{HTML_ESCAPE_PATTERN}/o
           text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
         end
@@ -231,53 +236,49 @@ module Encoders
         else
           @out << text
         end
-      else
-        
-        case text
-        
-        # token groups, eg. strings
-        when :open
-          @opened[0] = type
-          @out << (@css_style[@opened] || '<span>')
-          @opened << type
-        when :close
-          if @opened.empty?
-            # nothing to close
-          else
-            if $DEBUG and (@opened.size == 1 or @opened.last != type)
-              raise 'Malformed token stream: Trying to close a token (%p) \
-                that is not open. Open are: %p.' % [type, @opened[1..-1]]
-            end
-            @out << '</span>'
-            @opened.pop
-          end
         
-        # whole lines to be highlighted, eg. a deleted line in a diff
-        when :begin_line
-          @opened[0] = type
-          if style = @css_style[@opened]
-            @out << style.sub('<span', '<div')
-          else
-            @out << '<div>'
-          end
-          @opened << type
-        when :end_line
-          if @opened.empty?
-            # nothing to close
-          else
-            if $DEBUG and (@opened.size == 1 or @opened.last != type)
-              raise 'Malformed token stream: Trying to close a line (%p) \
-                that is not open. Open are: %p.' % [type, @opened[1..-1]]
-            end
-            @out << '</div>'
-            @opened.pop
+      
+      # token groups, eg. strings
+      when :open
+        @opened[0] = type
+        @out << (@css_style[@opened] || '<span>')
+        @opened << type
+      when :close
+        if @opened.empty?
+          # nothing to close
+        else
+          if $DEBUG and (@opened.size == 1 or @opened.last != type)
+            raise 'Malformed token stream: Trying to close a token (%p) \
+              that is not open. Open are: %p.' % [type, @opened[1..-1]]
           end
-        
-        when nil
-          raise 'Token with nil as text was given: %p' % [[text, type]]
+          @out << '</span>'
+          @opened.pop
+        end
+      
+      # whole lines to be highlighted, eg. a deleted line in a diff
+      when :begin_line
+        @opened[0] = type
+        if style = @css_style[@opened]
+          @out << style.sub('<span', '<div')
         else
-          raise 'unknown token kind: %p' % text
+          @out << '<div>'
         end
+        @opened << type
+      when :end_line
+        if @opened.empty?
+          # nothing to close
+        else
+          if $DEBUG and (@opened.size == 1 or @opened.last != type)
+            raise 'Malformed token stream: Trying to close a line (%p) \
+              that is not open. Open are: %p.' % [type, @opened[1..-1]]
+          end
+          @out << '</div>'
+          @opened.pop
+        end
+      
+      else
+        raise 'unknown token kind: %p' % [text]
+        
       end
     end
 

data/lib/coderay/encoders/html/output.rb

@@ -177,7 +177,7 @@ module Encoders
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
 <head>
   <meta http-equiv="content-type" content="text/html; charset=utf-8" />
-  <title>CodeRay HTML Encoder Example</title>
+  <title>CodeRay Output</title>
   <style type="text/css">
 <%CSS%>
   </style>

data/lib/coderay/helpers/file_type.rb

@@ -88,7 +88,12 @@ module FileType
     'cpp' => :c,
     'c' => :c,
     'h' => :c,
+    'java' => :java,
     'js' => :java_script,
+    'json' => :json,
+    'diff' => :diff,
+    'patch' => :diff,
+    'css' => :css,
     'xml' => :xml,
     'htm' => :html,
     'html' => :html,

data/lib/coderay/scanner.rb

@@ -4,7 +4,7 @@ module CodeRay
 
   # = Scanners
   #
-  # $Id: scanner.rb 253 2008-09-16 22:07:44Z murphy $
+  # $Id: scanner.rb 270 2009-01-01 03:19:28Z murphy $
   #
   # This module holds the Scanner class and its subclasses.
   # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
@@ -180,6 +180,11 @@ module CodeRay
       def line
         string[0..pos].count("\n") + 1
       end
+      
+      def column pos = self.pos
+        return 0 if pos <= 0
+        pos - (string.rindex(?\n, pos) || 0)
+      end
 
     protected
 
@@ -216,7 +221,7 @@ module CodeRay
 tokens:
 %s
 
-current line: %d  pos = %d
+current line: %d  column: %d  pos: %d
 matched: %p  state: %p
 bol? = %p,  eos? = %p
 
@@ -231,10 +236,10 @@ surrounding code:
           msg,
           tokens.size,
           tokens.last(10).map { |t| t.inspect }.join("\n"),
-          line, pos,
+          line, column, pos,
           matched, state, bol?, eos?,
-          string[pos-ambit,ambit],
-          string[pos,ambit],
+          string[pos - ambit, ambit],
+          string[pos, ambit],
         ]
       end
 

data/lib/coderay/scanners/_map.rb

@@ -7,7 +7,8 @@ module Scanners
     :irb => :ruby,
     :xhtml => :nitro_xhtml,
     :javascript => :java_script,
-    :nitro => :nitro_xhtml
+    :nitro => :nitro_xhtml,
+    :yml => :yaml
 
   default :plain
 

data/lib/coderay/scanners/css.rb

@@ -168,7 +168,6 @@ module Scanners
         raise_inspect 'Empty token', tokens unless match
 
         tokens << [match, kind]
-        # tokens << [states.inspect, :error]
 
       end
 

data/lib/coderay/scanners/html.rb

@@ -2,8 +2,6 @@ module CodeRay
 module Scanners
 
   # HTML Scanner
-  #
-  # $Id$
   class HTML < Scanner
 
     include Streamable
@@ -65,7 +63,7 @@ module Scanners
             if scan(/<!--.*?-->/m)
               kind = :comment
             elsif scan(/<!DOCTYPE.*?>/m)
-              kind = :preprocessor
+              kind = :doctype
             elsif scan(/<\?xml.*?\?>/m)
               kind = :preprocessor
             elsif scan(/<\?.*?\?>|<%.*?%>/m)

data/lib/coderay/scanners/json.rb

@@ -34,11 +34,11 @@ module Scanners
           elsif match = scan(/ [:,\[{\]}] /x)
             kind = :operator
             case match
-            when '{': stack << :object; key_expected = true
-            when '[': stack << :array
-            when ':': key_expected = false
-            when ',': key_expected = true if stack.last == :object
-            when '}', ']': stack.pop  # no error recovery, but works for valid JSON
+            when '{' then stack << :object; key_expected = true
+            when '[' then stack << :array
+            when ':' then key_expected = false
+            when ',' then key_expected = true if stack.last == :object
+            when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
             end
           elsif match = scan(/ true | false | null /x)
             kind = IDENT_KIND[match]

data/lib/coderay/scanners/ruby/patterns.rb

@@ -166,13 +166,14 @@ module Scanners
         { }
       ] ]
 
-      CLOSING_PAREN.values.each { |o| o.freeze }  # debug, if I try to change it with <<
+      CLOSING_PAREN.each { |k,v| k.freeze; v.freeze }  # debug, if I try to change it with <<
       OPENING_PAREN = CLOSING_PAREN.invert
 
       STRING_PATTERN = Hash.new { |h, k|
         delim, interpreted = *k
-        delim_pattern = Regexp.escape(delim.dup)
+        delim_pattern = Regexp.escape(delim)
         if closing_paren = CLOSING_PAREN[delim]
+          delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION  # JRuby fix
           delim_pattern << Regexp.escape(closing_paren)
         end
 

data/lib/coderay/scanners/yaml.rb

@@ -0,0 +1,141 @@
+module CodeRay
+module Scanners
+  
+  # YAML Scanner
+  #
+  # Based on the YAML scanner from Syntax by Jamis Buck.
+  class YAML < Scanner
+    
+    register_for :yaml
+    file_extension 'yml'
+    
+    def scan_tokens tokens, options
+      
+      value_expected = nil
+      state = :initial
+      key_indent = indent = 0
+      
+      until eos?
+        
+        kind = nil
+        match = nil
+        
+        if bol?
+          key_indent = nil
+          if $DEBUG
+            indent = check(/ +/) ? matched.size : 0
+            tokens << [indent.to_s, :debug]
+          end
+        end
+        
+        if match = scan(/ +[\t ]*/)
+          kind = :space
+          
+        elsif match = scan(/\n+/)
+          kind = :space
+          state = :initial if match.index(?\n)
+          
+        elsif match = scan(/#.*/)
+          kind = :comment
+          
+        elsif bol? and case
+          when match = scan(/---|\.\.\./)
+            tokens << [:open, :head]
+            tokens << [match, :head]
+            tokens << [:close, :head]
+            next
+          when match = scan(/%.*/)
+            tokens << [match, :doctype]
+            next
+          end
+        
+        elsif state == :value and case
+          when !check(/(?:"[^"]*")(?=: |:$)/) && scan(/"/)
+            tokens << [:open, :string]
+            tokens << [matched, :delimiter]
+            tokens << [matched, :content] if scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
+            tokens << [matched, :delimiter] if scan(/"/)
+            tokens << [:close, :string]
+            next
+          when match = scan(/[|>][-+]?/)
+            tokens << [:open, :string]
+            tokens << [match, :delimiter]
+            tokens << [matched, :content] if scan(/(?:\n+ {#{key_indent + 1}}.*)+/)
+            tokens << [:close, :string]
+            next
+          when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
+            tokens << [match, :string]
+            string_indent = key_indent || column(pos - match.size - 1)
+            tokens << [matched, :string] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+            next
+          end
+          
+        elsif case
+          when match = scan(/[-:](?= |$)/)
+            state = :value if state == :colon && (match == ':' || match == '-')
+            state = :value if state == :initial && match == '-'
+            kind = :operator
+          when match = scan(/[,{}\[\]]/)
+            kind = :operator
+          when state == :initial && match = scan(/[\w.() ]*\S(?=: |:$)/)
+            kind = :key
+            key_indent = column(pos - match.size - 1)
+            # tokens << [key_indent.inspect, :debug]
+            state = :colon
+          when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?=: |:$)/)
+            tokens << [:open, :key]
+            tokens << [match[0,1], :delimiter]
+            tokens << [match[1..-2], :content]
+            tokens << [match[-1,1], :delimiter]
+            tokens << [:close, :key]
+            key_indent = column(pos - match.size - 1)
+            # tokens << [key_indent.inspect, :debug]
+            state = :colon
+            next
+          when scan(/(![\w\/]+)(:([\w:]+))?/)
+            tokens << [self[1], :type]
+            if self[2]
+              tokens << [':', :operator]
+              tokens << [self[3], :class]
+            end
+            next
+          when scan(/&\S+/)
+            kind = :variable
+          when scan(/\*\w+/)
+            kind = :global_variable
+          when scan(/<</)
+            kind = :class_variable
+          when scan(/\d\d:\d\d:\d\d/)
+            kind = :oct
+          when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
+            kind = :oct
+          when scan(/:\w+/)
+            kind = :symbol
+          when scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
+            kind = :error
+          when scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
+            kind = :error
+          end
+          
+        else
+          getch
+          kind = :error
+          
+        end
+        
+        match ||= matched
+        
+        raise_inspect 'Error token %p in line %d' % [[match, kind], line], tokens if $DEBUG && !kind
+        raise_inspect 'Empty token', tokens unless match
+        
+        tokens << [match, kind]
+        
+      end
+      
+      tokens
+    end
+    
+  end
+  
+end
+end

data/lib/coderay/styles/cycnus.rb

@@ -64,6 +64,7 @@ ol.CodeRay li { white-space: pre }
 .di { color:#088; font-weight:bold }
 .dl { color:black }
 .do { color:#970 }
+.dt { color:#34b }
 .ds { color:#D42; font-weight:bold }
 .e  { color:#666; font-weight:bold }
 .en { color:#800; font-weight:bold }
@@ -107,15 +108,15 @@ ol.CodeRay li { white-space: pre }
 .rx .mod { color:#C2C }
 .rx .fu  { color:#404; font-weight: bold }
 
-.s { background-color:#fff0f0 }
+.s { background-color:#fff0f0; color: #D20; }
 .s .s { background-color:#ffe0e0 }
 .s .s  .s { background-color:#ffd0d0 }
-.s .k { color: #D20; }
+.s .k { }
 .s .ch { color: #b0b; }
 .s .dl { color: #710; }
 
-.sh { background-color:#f0fff0 }
-.sh .k { color:#2B2 }
+.sh { background-color:#f0fff0; color:#2B2 }
+.sh .k { }
 .sh .dl { color:#161 }
 
 .sy { color:#A60 }

data/lib/coderay/token_classes.rb

@@ -21,6 +21,7 @@ module CodeRay
       :delimiter => 'dl',
       :directive => 'di',
       :doc => 'do',
+      :doctype => 'dt',
       :doc_string => 'ds',
       :entity => 'en',
       :error => 'er',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: coderay
 version: !ruby/object:Gem::Version 
-  version: 0.8.260
+  version: 0.8.263
 platform: ruby
 authors: 
 - murphy
@@ -9,7 +9,7 @@ autorequire: coderay
 bindir: bin
 cert_chain: []
 
-date: 2008-10-08 00:00:00 +02:00
+date: 2009-01-01 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -68,6 +68,7 @@ files:
 - ./lib/coderay/scanners/sql.Keith.rb
 - ./lib/coderay/scanners/sql.rb
 - ./lib/coderay/scanners/xml.rb
+- ./lib/coderay/scanners/yaml.rb
 - ./lib/coderay/style.rb
 - ./lib/coderay/styles/_map.rb
 - ./lib/coderay/styles/cycnus.rb
@@ -104,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: 
 - strscan
 rubyforge_project: coderay
-rubygems_version: 1.3.0
+rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: CodeRay is a fast syntax highlighter engine for many languages.