raldred-coderay 0.9.0 → 0.9.339

data/lib/coderay/encoder.rb

@@ -0,0 +1,187 @@
+require "stringio"
+
+module CodeRay
+
+  # This module holds the Encoder class and its subclasses.
+  # For example, the HTML encoder is named CodeRay::Encoders::HTML
+  # can be found in coderay/encoders/html.
+  #
+  # Encoders also provides methods and constants for the register
+  # mechanism and the [] method that returns the Encoder class
+  # belonging to the given format.
+  module Encoders
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'encoders'
+
+    # = Encoder
+    #
+    # The Encoder base class. Together with Scanner and
+    # Tokens, it forms the highlighting triad.
+    #
+    # Encoder instances take a Tokens object and do something with it.
+    #
+    # The most common Encoder is surely the HTML encoder
+    # (CodeRay::Encoders::HTML). It highlights the code in a colorful
+    # html page.
+    # If you want the highlighted code in a div or a span instead,
+    # use its subclasses Div and Span.
+    class Encoder
+      extend Plugin
+      plugin_host Encoders
+
+      attr_reader :token_stream
+
+      class << self
+
+        # Returns if the Encoder can be used in streaming mode.
+        def streamable?
+          is_a? Streamable
+        end
+
+        # If FILE_EXTENSION isn't defined, this method returns the
+        # downcase class name instead.
+        def const_missing sym
+          if sym == :FILE_EXTENSION
+            plugin_id
+          else
+            super
+          end
+        end
+
+      end
+
+      # Subclasses are to store their default options in this constant.
+      DEFAULT_OPTIONS = { :stream => false }
+
+      # The options you gave the Encoder at creating.
+      attr_accessor :options
+
+      # Creates a new Encoder.
+      # +options+ is saved and used for all encode operations, as long
+      # as you don't overwrite it there by passing additional options.
+      #
+      # Encoder objects provide three encode methods:
+      # - encode simply takes a +code+ string and a +lang+
+      # - encode_tokens expects a +tokens+ object instead
+      # - encode_stream is like encode, but uses streaming mode.
+      #
+      # Each method has an optional +options+ parameter. These are
+      # added to the options you passed at creation.
+      def initialize options = {}
+        @options = self.class::DEFAULT_OPTIONS.merge options
+        raise "I am only the basic Encoder class. I can't encode "\
+          "anything. :( Use my subclasses." if self.class == Encoder
+      end
+
+      # Encode a Tokens object.
+      def encode_tokens tokens, options = {}
+        options = @options.merge options
+        setup options
+        compile tokens, options
+        finish options
+      end
+
+      # Encode the given +code+ after tokenizing it using the Scanner
+      # for +lang+.
+      def encode code, lang, options = {}
+        options = @options.merge options
+        scanner_options = CodeRay.get_scanner_options(options)
+        tokens = CodeRay.scan code, lang, scanner_options
+        encode_tokens tokens, options
+      end
+
+      # You can use highlight instead of encode, if that seems
+      # more clear to you.
+      alias highlight encode
+
+      # Encode the given +code+ using the Scanner for +lang+ in
+      # streaming mode.
+      def encode_stream code, lang, options = {}
+        raise NotStreamableError, self unless kind_of? Streamable
+        options = @options.merge options
+        setup options
+        scanner_options = CodeRay.get_scanner_options options
+        @token_stream =
+          CodeRay.scan_stream code, lang, scanner_options, &self
+        finish options
+      end
+
+      # Behave like a proc. The token method is converted to a proc.
+      def to_proc
+        method(:token).to_proc
+      end
+
+      # Return the default file extension for outputs of this encoder.
+      def file_extension
+        self.class::FILE_EXTENSION
+      end
+
+    protected
+
+      # Called with merged options before encoding starts.
+      # Sets @out to an empty string.
+      #
+      # See the HTML Encoder for an example of option caching.
+      def setup options
+        @out = ''
+      end
+
+      # Called with +text+ and +kind+ of the currently scanned token.
+      # For simple scanners, it's enougth to implement this method.
+      #
+      # By default, it calls text_token or block_token, depending on
+      # whether +text+ is a String.
+      def token text, kind
+        out =
+          if text.is_a? ::String
+            text_token text, kind
+          elsif text.is_a? ::Symbol
+            block_token text, kind
+          else
+            raise 'Unknown token text type: %p' % text
+          end
+        @out << out if defined?(@out) && @out
+      end
+
+      def text_token text, kind
+      end
+
+      def block_token action, kind
+        case action
+        when :open
+          open_token kind
+        when :close
+          close_token kind
+        when :begin_line
+          begin_line kind
+        when :end_line
+          end_line kind
+        else
+          raise 'unknown block action: %p' % action
+        end
+      end
+
+      # Called with merged options after encoding starts.
+      # The return value is the result of encoding, typically @out.
+      def finish options
+        @out
+      end
+
+      # Do the encoding.
+      #
+      # The already created +tokens+ object must be used; it can be a
+      # TokenStream or a Tokens object.
+      if RUBY_VERSION >= '1.9'
+        def compile tokens, options
+          tokens.each { |text, kind| token text, kind }  # FIXME for Ruby 1.9?
+        end
+      else
+        def compile tokens, options
+          tokens.each(&self)
+        end
+      end
+
+    end
+
+  end
+end

data/lib/coderay/encoders/_map.rb

@@ -0,0 +1,9 @@
+module CodeRay
+module Encoders
+
+  map :stats => :statistic,
+    :plain => :text,
+    :tex => :latex
+
+end
+end

data/lib/coderay/encoders/count.rb

@@ -0,0 +1,21 @@
+module CodeRay
+module Encoders
+
+  class Count < Encoder
+
+    include Streamable
+    register_for :count
+
+    protected
+
+    def setup options
+      @out = 0
+    end
+
+    def token text, kind
+      @out += 1
+    end
+  end
+
+end
+end

data/lib/coderay/encoders/debug.rb

@@ -0,0 +1,49 @@
+module CodeRay
+module Encoders
+
+  # = Debug Encoder
+  #
+  # Fast encoder producing simple debug output.
+  #
+  # It is readable and diff-able and is used for testing.
+  #
+  # You cannot fully restore the tokens information from the
+  # output, because consecutive :space tokens are merged.
+  # Use Tokens#dump for caching purposes.
+  class Debug < Encoder
+
+    include Streamable
+    register_for :debug
+
+    FILE_EXTENSION = 'raydebug'
+
+  protected
+    def text_token text, kind
+      if kind == :space
+        text
+      else
+        text = text.gsub(/[)\\]/, '\\\\\0')  # escape ) and \
+        "#{kind}(#{text})"
+      end
+    end
+
+    def open_token kind
+      "#{kind}<"
+    end
+
+    def close_token kind
+      ">"
+    end
+
+    def begin_line kind
+      "#{kind}["
+    end
+
+    def end_line kind
+      "]"
+    end
+
+  end
+
+end
+end

data/lib/coderay/encoders/div.rb

@@ -0,0 +1,20 @@
+module CodeRay
+module Encoders
+
+  load :html
+
+  class Div < HTML
+
+    FILE_EXTENSION = 'div.html'
+
+    register_for :div
+
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+      :css => :style,
+      :wrap => :div,
+    })
+
+  end
+
+end
+end

data/lib/coderay/encoders/html.rb

@@ -0,0 +1,306 @@
+require "set"
+
+module CodeRay
+module Encoders
+
+  # = HTML Encoder
+  #
+  # This is CodeRay's most important highlighter:
+  # It provides save, fast XHTML generation and CSS support.
+  #
+  # == Usage
+  #
+  #  require 'coderay'
+  #  puts CodeRay.scan('Some /code/', :ruby).html  #-> a HTML page
+  #  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
+  #  #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+  #  puts CodeRay.scan('Some /code/', :ruby).span  #-> the same
+  #  
+  #  puts CodeRay.scan('Some code', :ruby).html(
+  #    :wrap => nil,
+  #    :line_numbers => :inline,
+  #    :css => :style
+  #  )
+  #  #-> <span class="no">1</span>  <span style="color:#036; font-weight:bold;">Some</span> code
+  #
+  # == Options
+  #
+  # === :tab_width
+  # Convert \t characters to +n+ spaces (a number.)
+  # Default: 8
+  #
+  # === :css
+  # How to include the styles; can be :class or :style.
+  #
+  # Default: :class
+  #
+  # === :wrap
+  # Wrap in :page, :div, :span or nil.
+  #
+  # You can also use Encoders::Div and Encoders::Span.
+  #
+  # Default: nil
+  #
+  # === :title
+  # 
+  # The title of the HTML page (works only when :wrap is set to :page.)
+  #
+  # Default: 'CodeRay output'
+  #
+  # === :line_numbers
+  # Include line numbers in :table, :inline, :list or nil (no line numbers)
+  #
+  # Default: nil
+  #
+  # === :line_number_start
+  # Where to start with line number counting.
+  #
+  # Default: 1
+  #
+  # === :bold_every
+  # Make every +n+-th number appear bold.
+  #
+  # Default: 10
+  #
+  # === :highlight_lines
+  # 
+  # Highlights certain line numbers now by using the :highlight_lines option.
+  # Can be any Enumerable, typically just an Array or Range, of numbers.
+  # 
+  # Bolding is deactivated when :highlight_lines is set. It only makes sense
+  # in combination with :line_numbers.
+  #
+  # Default: nil
+  #
+  # === :hint
+  # Include some information into the output using the title attribute.
+  # Can be :info (show token type on mouse-over), :info_long (with full path)
+  # or :debug (via inspect).
+  #
+  # Default: false
+  class HTML < Encoder
+
+    include Streamable
+    register_for :html
+
+    FILE_EXTENSION = 'html'
+
+    DEFAULT_OPTIONS = {
+      :tab_width => 8,
+
+      :level => :xhtml,
+      :css => :class,
+
+      :style => :cycnus,
+      :wrap => nil,
+      :title => 'CodeRay output',
+
+      :line_numbers => nil,
+      :line_number_start => 1,
+      :bold_every => 10,
+      :highlight_lines => nil,
+
+      :hint => false,
+    }
+
+    helper :output, :css
+
+    attr_reader :css
+
+  protected
+
+    HTML_ESCAPE = {  #:nodoc:
+      '&' => '&amp;',
+      '"' => '&quot;',
+      '>' => '&gt;',
+      '<' => '&lt;',
+    }
+
+    # This was to prevent illegal HTML.
+    # Strange chars should still be avoided in codes.
+    evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
+    evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
+    #ansi_chars = Array(0x7f..0xff)
+    #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
+    # \x9 (\t) and \xA (\n) not included
+    #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
+
+    TOKEN_KIND_TO_INFO = Hash.new { |h, kind|
+      h[kind] =
+        case kind
+        when :pre_constant
+          'Predefined constant'
+        else
+          kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+        end
+    }
+
+    TRANSPARENT_TOKEN_KINDS = [
+      :delimiter, :modifier, :content, :escape, :inline_delimiter,
+    ].to_set
+
+    # Generate a hint about the given +classes+ in a +hint+ style.
+    #
+    # +hint+ may be :info, :info_long or :debug.
+    def self.token_path_to_hint hint, classes
+      title =
+        case hint
+        when :info
+          TOKEN_KIND_TO_INFO[classes.first]
+        when :info_long
+          classes.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+        when :debug
+          classes.inspect
+        end
+      title ? " title=\"#{title}\"" : ''
+    end
+
+    def setup options
+      super
+
+      @HTML_ESCAPE = HTML_ESCAPE.dup
+      @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+
+      @opened = [nil]
+      @css = CSS.new options[:style]
+
+      hint = options[:hint]
+      if hint and not [:debug, :info, :info_long].include? hint
+        raise ArgumentError, "Unknown value %p for :hint; \
+          expected :info, :debug, false, or nil." % hint
+      end
+
+      case options[:css]
+
+      when :class
+        @css_style = Hash.new do |h, k|
+          c = CodeRay::Tokens::ClassOfKind[k.first]
+          if c == :NO_HIGHLIGHT and not hint
+            h[k.dup] = false
+          else
+            title = if hint
+              HTML.token_path_to_hint(hint, k[1..-1] << k.first)
+            else
+              ''
+            end
+            if c == :NO_HIGHLIGHT
+              h[k.dup] = '<span%s>' % [title]
+            else
+              h[k.dup] = '<span%s class="%s">' % [title, c]
+            end
+          end
+        end
+
+      when :style
+        @css_style = Hash.new do |h, k|
+          if k.is_a? ::Array
+            styles = k.dup
+          else
+            styles = [k]
+          end
+          type = styles.first
+          classes = styles.map { |c| Tokens::ClassOfKind[c] }
+          if classes.first == :NO_HIGHLIGHT and not hint
+            h[k] = false
+          else
+            styles.shift if TRANSPARENT_TOKEN_KINDS.include? styles.first
+            title = HTML.token_path_to_hint hint, styles
+            style = @css[*classes]
+            h[k] =
+              if style
+                '<span%s style="%s">' % [title, style]
+              else
+                false
+              end
+          end
+        end
+
+      else
+        raise ArgumentError, "Unknown value %p for :css." % options[:css]
+
+      end
+    end
+
+    def finish options
+      not_needed = @opened.shift
+      @out << '</span>' * @opened.size
+      unless @opened.empty?
+        warn '%d tokens still open: %p' % [@opened.size, @opened]
+      end
+
+      @out.extend Output
+      @out.css = @css
+      @out.numerize! options[:line_numbers], options
+      @out.wrap! options[:wrap]
+      @out.apply_title! options[:title]
+
+      super
+    end
+
+    def token text, type = :plain
+      case text
+      
+      when nil
+        # raise 'Token with nil as text was given: %p' % [[text, type]] 
+      
+      when String
+        if text =~ /#{HTML_ESCAPE_PATTERN}/o
+          text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+        end
+        @opened[0] = type
+        if text != "\n" && style = @css_style[@opened]
+          @out << style << text << '</span>'
+        else
+          @out << text
+        end
+        
+      
+      # token groups, eg. strings
+      when :open
+        @opened[0] = type
+        @out << (@css_style[@opened] || '<span>')
+        @opened << type
+      when :close
+        if @opened.empty?
+          # nothing to close
+        else
+          if $DEBUG and (@opened.size == 1 or @opened.last != type)
+            raise 'Malformed token stream: Trying to close a token (%p) \
+              that is not open. Open are: %p.' % [type, @opened[1..-1]]
+          end
+          @out << '</span>'
+          @opened.pop
+        end
+      
+      # whole lines to be highlighted, eg. a deleted line in a diff
+      when :begin_line
+        @opened[0] = type
+        if style = @css_style[@opened]
+          @out << style.sub('<span', '<div')
+        else
+          @out << '<div>'
+        end
+        @opened << type
+      when :end_line
+        if @opened.empty?
+          # nothing to close
+        else
+          if $DEBUG and (@opened.size == 1 or @opened.last != type)
+            raise 'Malformed token stream: Trying to close a line (%p) \
+              that is not open. Open are: %p.' % [type, @opened[1..-1]]
+          end
+          @out << '</div>'
+          @opened.pop
+        end
+      
+      else
+        raise 'unknown token kind: %p' % [text]
+        
+      end
+    end
+
+  end
+
+end
+end