giga-fast-rb 0.0.1

data/coderay-1.1.3/lib/coderay/encoders/html.rb

@@ -0,0 +1,333 @@
+require 'set'
+
+module CodeRay
+module Encoders
+  
+  # = HTML Encoder
+  #
+  # This is CodeRay's most important highlighter:
+  # It provides save, fast XHTML generation and CSS support.
+  #
+  # == Usage
+  #
+  #  require 'coderay'
+  #  puts CodeRay.scan('Some /code/', :ruby).html  #-> a HTML page
+  #  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span)
+  #  #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+  #  puts CodeRay.scan('Some /code/', :ruby).span  #-> the same
+  #  
+  #  puts CodeRay.scan('Some code', :ruby).html(
+  #    :wrap => nil,
+  #    :line_numbers => :inline,
+  #    :css => :style
+  #  )
+  #
+  # == Options
+  #
+  # === :tab_width
+  # Convert \t characters to +n+ spaces (a number or false.)
+  # false will keep tab characters untouched.
+  # 
+  # Default: 8
+  #
+  # === :css
+  # How to include the styles; can be :class or :style.
+  #
+  # Default: :class
+  #
+  # === :wrap
+  # Wrap in :page, :div, :span or nil.
+  #
+  # You can also use Encoders::Div and Encoders::Span.
+  #
+  # Default: nil
+  #
+  # === :title
+  # 
+  # The title of the HTML page (works only when :wrap is set to :page.)
+  #
+  # Default: 'CodeRay output'
+  #
+  # === :break_lines
+  # 
+  # Split multiline blocks at line breaks.
+  # Forced to true if :line_numbers option is set to :inline.
+  #
+  # Default: false
+  #
+  # === :line_numbers
+  # Include line numbers in :table, :inline, or nil (no line numbers)
+  #
+  # Default: nil
+  #
+  # === :line_number_anchors
+  # Adds anchors and links to the line numbers. Can be false (off), true (on),
+  # or a prefix string that will be prepended to the anchor name.
+  #
+  # The prefix must consist only of letters, digits, and underscores.
+  #
+  # Default: true, default prefix name: "line"
+  #
+  # === :line_number_start
+  # Where to start with line number counting.
+  #
+  # Default: 1
+  #
+  # === :bold_every
+  # Make every +n+-th number appear bold.
+  #
+  # Default: 10
+  #
+  # === :highlight_lines
+  # 
+  # Highlights certain line numbers.
+  # Can be any Enumerable, typically just an Array or Range, of numbers.
+  # 
+  # Bolding is deactivated when :highlight_lines is set. It only makes sense
+  # in combination with :line_numbers.
+  #
+  # Default: nil
+  #
+  # === :hint
+  # Include some information into the output using the title attribute.
+  # Can be :info (show token kind on mouse-over), :info_long (with full path)
+  # or :debug (via inspect).
+  #
+  # Default: false
+  class HTML < Encoder
+    
+    register_for :html
+    
+    FILE_EXTENSION = 'snippet.html'
+    
+    DEFAULT_OPTIONS = {
+      :tab_width => 8,
+      
+      :css   => :class,
+      :style => :alpha,
+      :wrap  => nil,
+      :title => 'CodeRay output',
+      
+      :break_lines => false,
+      
+      :line_numbers        => nil,
+      :line_number_anchors => 'n',
+      :line_number_start   => 1,
+      :bold_every          => 10,
+      :highlight_lines     => nil,
+      
+      :hint => false,
+    }
+    
+    autoload :Output,    CodeRay.coderay_path('encoders', 'html', 'output')
+    autoload :CSS,       CodeRay.coderay_path('encoders', 'html', 'css')
+    autoload :Numbering, CodeRay.coderay_path('encoders', 'html', 'numbering')
+    
+    attr_reader :css
+    
+  protected
+    
+    def self.make_html_escape_hash
+      {
+        '&' => '&amp;',
+        '"' => '&quot;',
+        '>' => '&gt;',
+        '<' => '&lt;',
+        # "\t" => will be set to ' ' * options[:tab_width] during setup
+      }.tap do |hash|
+        # Escape ASCII control codes except \x9 == \t and \xA == \n.
+        (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
+      end
+    end
+    
+    HTML_ESCAPE = make_html_escape_hash
+    HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
+    
+    TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
+      h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
+    end
+    
+    TRANSPARENT_TOKEN_KINDS = Set[
+      :delimiter, :modifier, :content, :escape, :inline_delimiter,
+    ]
+    
+    # Generate a hint about the given +kinds+ in a +hint+ style.
+    #
+    # +hint+ may be :info, :info_long or :debug.
+    def self.token_path_to_hint hint, kinds
+      kinds = Array kinds
+      title =
+        case hint
+        when :info
+          kinds = kinds[1..-1] if TRANSPARENT_TOKEN_KINDS.include? kinds.first
+          TOKEN_KIND_TO_INFO[kinds.first]
+        when :info_long
+          kinds.reverse.map { |kind| TOKEN_KIND_TO_INFO[kind] }.join('/')
+        when :debug
+          kinds.inspect
+        end
+      title ? " title=\"#{title}\"" : ''
+    end
+    
+    def setup options
+      super
+      
+      check_options! options
+      
+      if options[:wrap] || options[:line_numbers]
+        @real_out = @out
+        @out = ''.dup
+      end
+      
+      @break_lines = (options[:break_lines] == true)
+      
+      @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => options[:tab_width] ? ' ' * options[:tab_width] : "\t")
+      
+      @opened = []
+      @last_opened = nil
+      @css = CSS.new options[:style]
+      
+      @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
+      
+      @set_last_opened = options[:hint] || options[:css] == :style
+    end
+    
+    def finish options
+      unless @opened.empty?
+        @out << '</span>' while @opened.pop
+        @last_opened = nil
+      end
+      
+      if @out.respond_to? :to_str
+        @out.extend Output
+        @out.css = @css
+        if options[:line_numbers]
+          Numbering.number! @out, options[:line_numbers], options
+        end
+        @out.wrap! options[:wrap]
+        @out.apply_title! options[:title]
+      end
+      
+      if defined?(@real_out) && @real_out
+        @real_out << @out
+        @out = @real_out
+      end
+      
+      super
+    end
+    
+  public
+    
+    def text_token text, kind
+      style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
+      
+      text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
+      text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
+      
+      if style
+        @out << style << text << '</span>'
+      else
+        @out << text
+      end
+    end
+    
+    # token groups, eg. strings
+    def begin_group kind
+      @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
+      @opened << kind
+      @last_opened = kind if @set_last_opened
+    end
+    
+    def end_group kind
+      check_group_nesting 'token group', kind if $CODERAY_DEBUG
+      close_span
+    end
+    
+    # whole lines to be highlighted, eg. a deleted line in a diff
+    def begin_line kind
+      if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
+        if style['class="']
+          @out << style.sub('class="', 'class="line ')
+        else
+          @out << style.sub('>', ' class="line">')
+        end
+      else
+        @out << '<span class="line">'
+      end
+      @opened << kind
+      @last_opened = kind if @options[:css] == :style
+    end
+    
+    def end_line kind
+      check_group_nesting 'line', kind if $CODERAY_DEBUG
+      close_span
+    end
+    
+  protected
+    
+    def check_options! options
+      unless [false, nil, :debug, :info, :info_long].include? options[:hint]
+        raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
+      end
+      
+      unless [:class, :style].include? options[:css]
+        raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
+      end
+      
+      options[:break_lines] = true if options[:line_numbers] == :inline
+    end
+    
+    def css_class_for_kinds kinds
+      TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
+    end
+    
+    def style_for_kinds kinds
+      css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
+      @css.get_style_for_css_classes css_classes
+    end
+    
+    def make_span_for_kinds method, hint
+      Hash.new do |h, kinds|
+        begin
+          css_class = css_class_for_kinds(kinds)
+          title     = HTML.token_path_to_hint hint, kinds if hint
+          
+          if css_class || title
+            if method == :style
+              style = style_for_kinds(kinds)
+              "<span#{title}#{" style=\"#{style}\"" if style}>"
+            else
+              "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
+            end
+          end
+        end.tap do |span|
+          h.clear if h.size >= 100
+          h[kinds] = span
+        end
+      end
+    end
+    
+    def check_group_nesting name, kind
+      if @opened.empty? || @opened.last != kind
+        warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
+      end
+    end
+    
+    def break_lines text, style
+      reopen = ''.dup
+      @opened.each_with_index do |kind, index|
+        reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '<span>')
+      end
+      text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
+    end
+    
+    def close_span
+      if @opened.pop
+        @out << '</span>'
+        @last_opened = @opened.last if @last_opened
+      end
+    end
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/json.rb

@@ -0,0 +1,83 @@
+module CodeRay
+module Encoders
+  
+  # A simple JSON Encoder.
+  # 
+  # Example:
+  #  CodeRay.scan('puts "Hello world!"', :ruby).json
+  # yields
+  #  [
+  #    {"type"=>"text", "text"=>"puts", "kind"=>"ident"},
+  #    {"type"=>"text", "text"=>" ", "kind"=>"space"},
+  #    {"type"=>"block", "action"=>"open", "kind"=>"string"},
+  #    {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+  #    {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
+  #    {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+  #    {"type"=>"block", "action"=>"close", "kind"=>"string"},
+  #  ]
+  class JSON < Encoder
+    
+    begin
+      require 'json'
+    rescue LoadError
+      begin
+        require 'rubygems' unless defined? Gem
+        gem 'json'
+        require 'json'
+      rescue LoadError
+        $stderr.puts "The JSON encoder needs the JSON library.\n" \
+          "Please gem install json."
+        raise
+      end
+    end
+    
+    register_for :json
+    FILE_EXTENSION = 'json'
+    
+  protected
+    def setup options
+      super
+      
+      @first = true
+      @out << '['
+    end
+    
+    def finish options
+      @out << ']'
+    end
+    
+    def append data
+      if @first
+        @first = false
+      else
+        @out << ','
+      end
+      
+      @out << data.to_json
+    end
+    
+  public
+    def text_token text, kind
+      append :type => 'text', :text => text, :kind => kind
+    end
+    
+    def begin_group kind
+      append :type => 'block', :action => 'open', :kind => kind
+    end
+    
+    def end_group kind
+      append :type => 'block', :action => 'close', :kind => kind
+    end
+    
+    def begin_line kind
+      append :type => 'block', :action => 'begin_line', :kind => kind
+    end
+    
+    def end_line kind
+      append :type => 'block', :action => 'end_line', :kind => kind
+    end
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/lines_of_code.rb

@@ -0,0 +1,45 @@
+module CodeRay
+module Encoders
+  
+  # Counts the LoC (Lines of Code). Returns an Integer >= 0.
+  # 
+  # Alias: +loc+
+  # 
+  # Everything that is not comment, markup, doctype/shebang, or an empty line,
+  # is considered to be code.
+  # 
+  # For example,
+  # * HTML files not containing JavaScript have 0 LoC
+  # * in a Java class without comments, LoC is the number of non-empty lines
+  # 
+  # A Scanner class should define the token kinds that are not code in the
+  # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
+  class LinesOfCode < TokenKindFilter
+    
+    register_for :lines_of_code
+    
+    NON_EMPTY_LINE = /^\s*\S.*$/
+    
+  protected
+    
+    def setup options
+      if scanner
+        kinds_not_loc = scanner.class::KINDS_NOT_LOC
+      else
+        warn "Tokens have no associated scanner, counting all nonempty lines." if $VERBOSE
+        kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
+      end
+      
+      options[:exclude] = kinds_not_loc
+      
+      super options
+    end
+    
+    def finish options
+      output @tokens.text.scan(NON_EMPTY_LINE).size
+    end
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/lint.rb

@@ -0,0 +1,59 @@
+module CodeRay
+module Encoders
+  
+  # = Lint Encoder
+  #
+  # Checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::DebugLint
+  class Lint < Debug
+    
+    register_for :lint
+    
+    InvalidTokenStream         = Class.new StandardError
+    EmptyToken                 = Class.new InvalidTokenStream
+    UnknownTokenKind           = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def text_token text, kind
+      raise EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
+    end
+    
+    def begin_group kind
+      @opened << kind
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    def begin_line kind
+      @opened << kind
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    protected
+    
+    def setup options
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
+    end
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/null.rb

@@ -0,0 +1,18 @@
+module CodeRay
+module Encoders
+  
+  # = Null Encoder
+  #
+  # Does nothing and returns an empty string.
+  class Null < Encoder
+    
+    register_for :null
+    
+    def text_token text, kind
+      # do nothing
+    end
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/page.rb

@@ -0,0 +1,24 @@
+module CodeRay
+module Encoders
+  
+  load :html
+  
+  # Wraps the output into a HTML page, using CSS classes and
+  # line numbers in the table format by default.
+  # 
+  # See Encoders::HTML for available options.
+  class Page < HTML
+    
+    FILE_EXTENSION = 'html'
+    
+    register_for :page
+    
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css          => :class,
+      :wrap         => :page,
+      :line_numbers => :table
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/span.rb

@@ -0,0 +1,23 @@
+module CodeRay
+module Encoders
+  
+  load :html
+  
+  # Wraps HTML output into a SPAN element, using inline styles by default.
+  # 
+  # See Encoders::HTML for available options.
+  class Span < HTML
+    
+    FILE_EXTENSION = 'span.html'
+    
+    register_for :span
+    
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css          => :style,
+      :wrap         => :span,
+      :line_numbers => false
+    
+  end
+  
+end
+end

data/coderay-1.1.3/lib/coderay/encoders/statistic.rb

@@ -0,0 +1,95 @@
+module CodeRay
+module Encoders
+  
+  # Makes a statistic for the given tokens.
+  # 
+  # Alias: +stats+
+  class Statistic < Encoder
+    
+    register_for :statistic
+    
+    attr_reader :type_stats, :real_token_count  # :nodoc:
+    
+    TypeStats = Struct.new :count, :size  # :nodoc:
+    
+  protected
+    
+    def setup options
+      super
+      
+      @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+      @real_token_count = 0
+    end
+    
+    STATS = <<-STATS  # :nodoc:
+
+Code Statistics
+
+Tokens            %8d
+  Non-Whitespace  %8d
+Bytes Total       %8d
+
+Token Types (%d):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+%s
+    STATS
+    
+    TOKEN_TYPES_ROW = <<-TKR  # :nodoc:
+  %-20s  %8d  %6.2f %%   %5.1f
+    TKR
+    
+    def finish options
+      all = @type_stats['TOTAL']
+      all_count, all_size = all.count, all.size
+      @type_stats.each do |type, stat|
+        stat.size /= stat.count.to_f
+      end
+      types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
+        TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+      end.join
+      @out << STATS % [
+        all_count, @real_token_count, all_size,
+        @type_stats.delete_if { |k, v| k.is_a? String }.size,
+        types_stats
+      ]
+      
+      super
+    end
+    
+  public
+    
+    def text_token text, kind
+      @real_token_count += 1 unless kind == :space
+      @type_stats[kind].count += 1
+      @type_stats[kind].size += text.size
+      @type_stats['TOTAL'].size += text.size
+      @type_stats['TOTAL'].count += 1
+    end
+    
+    def begin_group kind
+      block_token ':begin_group', kind
+    end
+    
+    def end_group kind
+      block_token ':end_group', kind
+    end
+    
+    def begin_line kind
+      block_token ':begin_line', kind
+    end
+    
+    def end_line kind
+      block_token ':end_line', kind
+    end
+    
+    def block_token action, kind
+      @type_stats['TOTAL'].count += 1
+      @type_stats[action].count += 1
+      @type_stats[kind].count += 1
+    end
+    
+  end
+  
+end
+end