coderay 1.0.0 → 1.0.0.598.pre

data/lib/coderay/token_kinds.rb

@@ -1,90 +1,89 @@
 module CodeRay
-  
-  # A Hash of all known token kinds and their associated CSS classes.
-  TokenKinds = Hash.new do |h, k|
-    warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
-    false
+  class Tokens
+    AbbreviationForKind = Hash.new do |h, k|  # :nodoc:
+      if $CODERAY_DEBUG
+        raise 'Undefined Token kind: %p' % [k]  # :nodoc:
+      else
+        :NO_HIGHLIGHT
+      end
+    end
+    AbbreviationForKind.update with = {  # :nodoc:
+      :annotation => 'at',
+      :attribute_name => 'an',
+      :attribute_value => 'av',
+      :bin => 'bi',
+      :char => 'ch',
+      :class => 'cl',
+      :class_variable => 'cv',
+      :color => 'cr',
+      :comment => 'c',
+      :complex => 'cm',
+      :constant => 'co',
+      :content => 'k',
+      :decorator => 'de',
+      :definition => 'df',
+      :delimiter => 'dl',
+      :directive => 'di',
+      :doc => 'do',
+      :doctype => 'dt',
+      :doc_string => 'ds',
+      :entity => 'en',
+      :error => 'er',
+      :escape => 'e',
+      :exception => 'ex',
+      :filename => 'filename',
+      :float => 'fl',
+      :function => 'fu',
+      :global_variable => 'gv',
+      :hex => 'hx',
+      :imaginary => 'cm',
+      :important => 'im',
+      :include => 'ic',
+      :inline => 'il',
+      :inline_delimiter => 'idl',
+      :instance_variable => 'iv',
+      :integer => 'i',
+      :interpreted => 'in',
+      :key => 'ke',
+      :keyword => 'kw',
+      :label => 'la',
+      :local_variable => 'lv',
+      :modifier => 'mod',
+      :namespace => 'ns',
+      :oct => 'oc',
+      :predefined => 'pd',
+      :preprocessor => 'pp',
+      :pre_constant => 'pc',
+      :pre_type => 'pt',
+      :pseudo_class => 'ps',
+      :regexp => 'rx',
+      :reserved => 'r',
+      :shell => 'sh',
+      :string => 's',
+      :symbol => 'sy',
+      :tag => 'ta',
+      :tag_special => 'ts',
+      :type => 'ty',
+      :value => 'vl',
+      :variable => 'v',
+      
+      :insert => 'ins',
+      :delete => 'del',
+      :change => 'chg',
+      :head => 'head',
+      
+      :eyecatcher => 'eye',
+      
+      :ident => :NO_HIGHLIGHT, # 'id'
+      #:operator => 'op',
+      :operator => :NO_HIGHLIGHT,  # 'op'
+      :space => :NO_HIGHLIGHT,  # 'sp'
+      :plain => :NO_HIGHLIGHT,
+    }
+    AbbreviationForKind[:method] = AbbreviationForKind[:function]
+    AbbreviationForKind[:nesting_delimiter] = AbbreviationForKind[:delimiter]
+    AbbreviationForKind[:escape] = AbbreviationForKind[:delimiter]
+    AbbreviationForKind[:docstring] = AbbreviationForKind[:comment]
+    #AbbreviationForKind.default = AbbreviationForKind[:error] or raise 'no class found for :error!'
   end
-  
-  # speedup
-  TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
-  
-  TokenKinds.update(  # :nodoc:
-    :annotation          => 'annotation',
-    :attribute_name      => 'attribute-name',
-    :attribute_value     => 'attribute-value',
-    :binary              => 'bin',
-    :char                => 'char',
-    :class               => 'class',
-    :class_variable      => 'class-variable',
-    :color               => 'color',
-    :comment             => 'comment',
-    :complex             => 'complex',
-    :constant            => 'constant',
-    :content             => 'content',
-    :debug               => 'debug',
-    :decorator           => 'decorator',
-    :definition          => 'definition',
-    :delimiter           => 'delimiter',
-    :directive           => 'directive',
-    :doc                 => 'doc',
-    :doctype             => 'doctype',
-    :doc_string          => 'doc-string',
-    :entity              => 'entity',
-    :error               => 'error',
-    :escape              => 'escape',
-    :exception           => 'exception',
-    :filename            => 'filename',
-    :float               => 'float',
-    :function            => 'function',
-    :global_variable     => 'global-variable',
-    :hex                 => 'hex',
-    :imaginary           => 'imaginary',
-    :important           => 'important',
-    :include             => 'include',
-    :inline              => 'inline',
-    :inline_delimiter    => 'inline-delimiter',
-    :instance_variable   => 'instance-variable',
-    :integer             => 'integer',
-    :key                 => 'key',
-    :keyword             => 'keyword',
-    :label               => 'label',
-    :local_variable      => 'local-variable',
-    :modifier            => 'modifier',
-    :namespace           => 'namespace',
-    :octal               => 'octal',
-    :predefined          => 'predefined',
-    :predefined_constant => 'predefined-constant',
-    :predefined_type     => 'predefined-type',
-    :preprocessor        => 'preprocessor',
-    :pseudo_class        => 'pseudo-class',
-    :regexp              => 'regexp',
-    :reserved            => 'reserved',
-    :shell               => 'shell',
-    :string              => 'string',
-    :symbol              => 'symbol',
-    :tag                 => 'tag',
-    :type                => 'type',
-    :value               => 'value',
-    :variable            => 'variable',
-    
-    :change              => 'change',
-    :delete              => 'delete',
-    :head                => 'head',
-    :insert              => 'insert',
-    
-    :eyecatcher          => 'eyecatcher',
-    
-    :ident               => false,
-    :operator            => false,
-    
-    :space               => false,
-    :plain               => false
-  )
-  
-  TokenKinds[:method]    = TokenKinds[:function]
-  TokenKinds[:escape]    = TokenKinds[:delimiter]
-  TokenKinds[:docstring] = TokenKinds[:comment]
-  
-  TokenKinds.freeze
 end

data/lib/coderay/tokens.rb

@@ -1,8 +1,5 @@
 module CodeRay
-  
-  # GZip library for writing and reading token dumps.
-  autoload :GZip, 'coderay/helpers/gzip'
-  
+
   # = Tokens  TODO: Rewrite!
   #
   # The Tokens class represents a list of tokens returnd from
@@ -64,25 +61,127 @@ module CodeRay
     #
     # options are passed to the encoder.
     def encode encoder, options = {}
-      encoder = Encoders[encoder].new options if encoder.respond_to? :to_sym
+      unless encoder.is_a? Encoders::Encoder
+        unless encoder.is_a? Class
+          encoder_class = Encoders[encoder]
+        end
+        encoder = encoder_class.new options
+      end
       encoder.encode_tokens self, options
     end
-    
-    # Turn tokens into a string by concatenating them.
-    def to_s
-      encode CodeRay::Encoders::Encoder.new
+
+    # Turn into a string using Encoders::Text.
+    #
+    # +options+ are passed to the encoder if given.
+    def to_s options = {}
+      encode :text, options
     end
-    
+
     # Redirects unknown methods to encoder calls.
     #
     # For example, if you call +tokens.html+, the HTML encoder
     # is used to highlight the tokens.
     def method_missing meth, options = {}
-      encode meth, options
+      encode_with meth, options
     rescue PluginHost::PluginNotFound
       super
     end
     
+    def encode_with encoder, options = {}
+      Encoders[encoder].new(options).encode_tokens self
+    end
+    
+    # Returns the tokens compressed by joining consecutive
+    # tokens of the same kind.
+    #
+    # This can not be undone, but should yield the same output
+    # in most Encoders.  It basically makes the output smaller.
+    #
+    # Combined with dump, it saves space for the cost of time.
+    #
+    # If the scanner is written carefully, this is not required -
+    # for example, consecutive //-comment lines could already be
+    # joined in one comment token by the Scanner.
+    def optimize
+      raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
+      last_kind = last_text = nil
+      new = self.class.new
+      for text, kind in self
+        if text.is_a? String
+          if kind == last_kind
+            last_text << text
+          else
+            new << [last_text, last_kind] if last_kind
+            last_text = text
+            last_kind = kind
+          end
+        else
+          new << [last_text, last_kind] if last_kind
+          last_kind = last_text = nil
+          new << [text, kind]
+        end
+      end
+      new << [last_text, last_kind] if last_kind
+      new
+    end
+
+    # Compact the object itself; see optimize.
+    def optimize!
+      replace optimize
+    end
+    
+    # Ensure that all begin_group tokens have a correspondent end_group.
+    #
+    # TODO: Test this!
+    def fix
+      raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
+      tokens = self.class.new
+      # Check token nesting using a stack of kinds.
+      opened = []
+      for type, kind in self
+        case type
+        when :begin_group
+          opened.push [:begin_group, kind]
+        when :begin_line
+          opened.push [:end_line, kind]
+        when :end_group, :end_line
+          expected = opened.pop
+          if [type, kind] != expected
+            # Unexpected end; decide what to do based on the kind:
+            # - token was never opened: delete the end (just skip it)
+            next unless opened.rindex expected
+            # - token was opened earlier: also close tokens in between
+            tokens << token until (token = opened.pop) == expected
+          end
+        end
+        tokens << [type, kind]
+      end
+      # Close remaining opened tokens
+      tokens << token while token = opened.pop
+      tokens
+    end
+    
+    def fix!
+      replace fix
+    end
+    
+    # TODO: Scanner#split_into_lines
+    # 
+    # Makes sure that:
+    # - newlines are single tokens
+    #   (which means all other token are single-line)
+    # - there are no open tokens at the end the line
+    #
+    # This makes it simple for encoders that work line-oriented,
+    # like HTML with list-style numeration.
+    def split_into_lines
+      raise NotImplementedError
+    end
+
+    def split_into_lines!
+      replace split_into_lines
+    end
+    
     # Split the tokens into parts of the given +sizes+.
     # 
     # The result will be an Array of Tokens objects. The parts have
@@ -121,15 +220,12 @@ module CodeRay
                 content_or_kind
               end
             end
-            part.concat closing
-            begin
-              parts << part
-              part = Tokens.new
-              size = sizes[i += 1]
-            end until size.nil? || size > 0
+            parts << part.concat(closing)
+            part = Tokens.new
             # ...and open them again.
             part.concat opened.flatten
             part_size = 0
+            size = sizes[i += 1]
             redo unless content.empty?
           else
             part << content << item
@@ -143,19 +239,19 @@ module CodeRay
           when :end_group, :end_line
             opened.pop
           else
-            raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item]
+            raise 'Unknown token action: %p, kind = %p' % [content, item]
           end
           part << content << item
           content = nil
         else
-          raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item]
+          raise 'else case reached'
         end
       end
       parts << part
       parts << Tokens.new while parts.size < sizes.size
       parts
     end
-    
+
     # Dumps the object into a String that can be saved
     # in files or databases.
     #
@@ -172,8 +268,9 @@ module CodeRay
     #
     # See GZip module.
     def dump gzip_level = 7
+      require 'coderay/helpers/gzip_simple'
       dump = Marshal.dump self
-      dump = GZip.gzip dump, gzip_level
+      dump = dump.gzip gzip_level
       dump.extend Undumping
     end
     
@@ -181,7 +278,7 @@ module CodeRay
     def count
       size / 2
     end
-    
+
     # Include this module to give an object an #undump
     # method.
     #
@@ -192,17 +289,18 @@ module CodeRay
         Tokens.load self
       end
     end
-    
+
     # Undump the object using Marshal.load, then
     # unzip it using GZip.gunzip.
     #
     # The result is commonly a Tokens object, but
     # this is not guaranteed.
     def Tokens.load dump
-      dump = GZip.gunzip dump
+      require 'coderay/helpers/gzip_simple'
+      dump = dump.gunzip
       @dump = Marshal.load dump
     end
-    
+
     alias text_token push
     def begin_group kind; push :begin_group, kind end
     def end_group kind; push :end_group, kind end
@@ -211,5 +309,50 @@ module CodeRay
     alias tokens concat
     
   end
-  
+
 end
+
+if $0 == __FILE__
+  $VERBOSE = true
+  $: << File.join(File.dirname(__FILE__), '..')
+  eval DATA.read, nil, $0, __LINE__ + 4
+end
+
+__END__
+require 'test/unit'
+
+class TokensTest < Test::Unit::TestCase
+  
+  def test_creation
+    assert CodeRay::Tokens < Array
+    tokens = nil
+    assert_nothing_raised do
+      tokens = CodeRay::Tokens.new
+    end
+    assert_kind_of Array, tokens
+  end
+  
+  def test_adding_tokens
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens.text_token 'string', :type
+      tokens.text_token '()', :operator
+    end
+    assert_equal tokens.size, 4
+    assert_equal tokens.count, 2
+  end
+  
+  def test_dump_undump
+    tokens = CodeRay::Tokens.new
+    assert_nothing_raised do
+      tokens.text_token 'string', :type
+      tokens.text_token '()', :operator
+    end
+    tokens2 = nil
+    assert_nothing_raised do
+      tokens2 = tokens.dump.undump
+    end
+    assert_equal tokens, tokens2
+  end
+  
+end