coderay 0.4.3.48

data/lib/coderay/encoders/html.rb

@@ -0,0 +1,169 @@
+module CodeRay
+module Encoders
+
+	class HTML < Encoder
+
+		include Streamable
+		register_for :html
+
+		FILE_EXTENSION = 'html'
+
+		DEFAULT_OPTIONS = {
+			:tab_width => 8,
+
+			:level => :xhtml,
+			:css => :class,
+
+			:wrap => :page,
+
+			:line_numbers => nil,
+			:line_number_start => 1,
+			:bold_every => 10,
+		}
+		NUMERIZABLE_WRAPPINGS = [:div, :page]
+		
+		require 'coderay/encoders/helpers/html_helper'
+		require 'coderay/encoders/helpers/html_output'
+		require 'coderay/encoders/helpers/html_css'
+
+		def initialize(*)
+			super
+			@last_options = nil
+		end
+
+	protected
+		
+		HTML_ESCAPE = {  #:nodoc:
+			'&' => '&amp;',
+			'"' => '&quot;',
+			'>' => '&gt;',
+			'<' => '&lt;',
+		}
+
+		# This is to prevent illegal HTML.
+		# Strange chars should still be avoided in codes.
+		evil_chars = Array(0x00...0x20) - [?n, ?t]
+		evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
+		ansi_chars = Array(0x7f..0xff)
+		ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
+		# \x9 (\t) and \xA (\n) not included
+		HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/
+
+		def setup options
+			if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap]
+				warn ':line_numbers wanted, but :wrap is %p' % options[:wrap]
+			end
+			super
+			return if options == @last_options
+			@last_options = options
+
+			@HTML_ESCAPE = HTML_ESCAPE.dup
+			@HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
+			
+			@opened = [nil]
+			@css = CSS.new
+
+			case options[:css]
+			
+			when :class
+				@css_style = Hash.new do |h, k|
+					if k.is_a? Array
+						type = k.first
+					else
+						type = k
+					end
+					c = ClassOfKind[type]
+					if c == :NO_HIGHLIGHT
+						h[k] = false
+					else
+						if options[:debug]
+							debug_info = ' title="%p"' % [ k ]
+						else
+							debug_info = ''
+						end
+						h[k] = '<span%s class="%s">' % [debug_info, c]
+					end
+				end
+				
+			when :style
+				@css_style = Hash.new do |h, k|
+					if k.is_a? Array
+						styles = k.dup
+					else
+						styles = [k]
+					end
+					styles.map! { |c| ClassOfKind[c] }
+					if styles.first == :NO_HIGHLIGHT
+						h[k] = false
+					else
+						if options[:debug]
+							debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ]
+						else
+							debug_info = ''
+						end
+						style = @css[*styles]
+						h[k] =
+							if style
+								'<span%s style="%s">' % [debug_info, style]
+							else
+								false
+							end
+					end
+				end
+				
+			else
+				raise "Unknown value %p for :css." % options[:css]
+				
+			end
+		end
+
+		def finish options
+			not_needed = @opened.shift
+			@out << '</span>' * @opened.size
+
+			@out.extend Output
+			@out.numerize! options[:line_numbers], options # if options[:line_numbers]
+			@out.wrap! options[:wrap] # if options[:wrap]
+
+			#require 'pp'
+			#pp @css_style, @css_style.size
+
+			super
+		end
+
+		def token text, type
+			if text.is_a? String
+				if text =~ /#{HTML_ESCAPE_PATTERN}/o
+					text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
+				end
+				@opened[0] = type
+				style = @css_style[@opened]
+				if style
+					@out << style << text << '</span>'
+				else
+					@out << text
+				end
+			else
+				case text
+				when :open
+					@opened[0] = type
+					@out << @css_style[@opened]
+					@opened << type
+				when :close
+					unless @opened.empty?
+						raise 'Not Token to be closed.' unless @opened.size > 1
+						@out << '</span>'
+						@opened.pop
+					end
+				when nil
+					raise 'Token with nil as text was given: %p' % [[text, type]]
+				else
+					raise 'unknown token kind: %p' % text
+				end
+			end
+		end
+		
+	end
+
+end
+end

data/lib/coderay/encoders/null.rb

@@ -0,0 +1,20 @@
+module CodeRay
+	module Encoders
+
+		class Null < Encoder
+
+			include Streamable
+			register_for :null
+
+			protected
+
+			def token(*)
+				# do nothing
+			end
+
+		end
+
+	end
+end
+
+

data/lib/coderay/encoders/span.rb

@@ -0,0 +1,16 @@
+module CodeRay module Encoders
+	
+	require 'coderay/encoders/html'
+	class Span < HTML
+
+		FILE_EXTENSION = 'span.html'
+
+		register_for :span
+
+		DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
+			:css => :style,
+			:wrap => :span,
+		})
+	end
+
+end end

data/lib/coderay/encoders/statistic.rb

@@ -0,0 +1,74 @@
+module CodeRay module Encoders
+
+	# Makes a statistic for the given tokens.
+	class Statistic < Encoder
+
+		include Streamable
+		register_for :stats, :statistic
+
+		attr_reader :type_stats, :real_token_count
+
+	protected
+		
+		TypeStats = Struct.new :count, :size
+
+		def setup options
+			@type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+			@real_token_count = 0
+		end
+
+		def generate tokens, options
+			@tokens = tokens
+			super
+		end
+		
+		def token text, type
+			@type_stats['TOTAL'].count += 1
+			if text.is_a? String
+				@real_token_count += 1 unless type == :space
+				@type_stats[type].count += 1
+				@type_stats[type].size += text.size
+				@type_stats['TOTAL'].size += text.size
+			else
+				@content_type = type
+				@type_stats['open/close'].count += 1
+			end
+		end
+
+		STATS = <<-STATS
+
+Code Statistics
+
+Tokens            %8d
+  Non-Whitespace  %8d
+Bytes Total       %8d
+
+Token Types (%d):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+%s
+			STATS
+# space                    12007   33.81 %     1.7
+		TOKEN_TYPES_ROW = <<-TKR
+  %-20s  %8d  %6.2f %%   %5.1f
+			TKR
+
+		def finish options
+			all = @type_stats['TOTAL']
+			all_count, all_size = all.count, all.size
+			@type_stats.each do |type, stat|
+				stat.size /= stat.count.to_f
+			end
+			types_stats = @type_stats.sort_by { |k, v| -v.count }.map do |k, v|
+				TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+			end.join
+			STATS % [
+				all_count, @real_token_count, all_size,
+				@type_stats.delete_if { |k, v| k.is_a? String }.size,
+				types_stats
+			]
+		end
+		
+	end
+
+end end

data/lib/coderay/encoders/text.rb

@@ -0,0 +1,33 @@
+module CodeRay
+module Encoders
+
+	class Text < Encoder
+
+		include Streamable
+		register_for :text
+
+		FILE_EXTENSION = 'txt'
+
+		DEFAULT_OPTIONS = {
+			:separator => ''
+		}
+
+	protected
+		def setup options
+			super
+			@sep = options[:separator]
+		end
+
+		def token text, kind
+			return unless text.respond_to? :to_str
+			@out << text + @sep
+		end
+
+		def finish options
+			@out.chomp @sep
+		end
+
+	end
+
+end
+end

data/lib/coderay/encoders/tokens.rb

@@ -0,0 +1,44 @@
+module CodeRay
+	module Encoders
+
+		# The Tokens encoder converts the tokens to a simple
+		# readable format. It doesn't use colors and is mainly
+		# intended for console output.
+		# 
+		# The tokens are converted with Tokens.write_token.
+		#
+		# The format is:
+		#
+		#   <token-kind> \t <escaped token-text> \n
+		#
+		# Example:
+		#
+		#   require 'coderay'
+		#   puts CodeRay.scan("puts 3 + 4", :ruby).tokens
+		# 
+		# prints:
+		#   
+		#   ident   puts
+		#   space
+		#   integer 3
+		#   space
+		#   operator        +
+		#   space
+		#   integer 4
+		# 
+		class Tokens < Encoder
+
+			include Streamable
+			register_for :tokens
+
+			FILE_EXTENSION = 'tok'
+
+			protected
+			def token *args
+				@out << CodeRay::Tokens.write_token(*args)
+			end
+
+		end
+
+	end
+end

data/lib/coderay/encoders/yaml.rb

@@ -0,0 +1,19 @@
+module CodeRay
+	module Encoders
+
+		class YAML < Encoder
+
+			register_for :yaml
+
+			FILE_EXTENSION = 'yaml'
+
+			protected
+			def compile tokens, options
+				require 'yaml'
+				@out = tokens.to_a.to_yaml
+			end
+
+		end
+
+	end
+end

data/lib/coderay/helpers/filetype.rb

@@ -0,0 +1,145 @@
+# =FileType
+#
+# A simple filetype recognizer
+#
+# Author: murphy (mail to murphy cYcnus de)
+#
+# Version: 0.1 (2005.september.1)
+#
+# ==Documentation
+#
+# TODO
+#
+module FileType
+	
+	UnknownFileType = Class.new Exception
+
+	class << self
+
+		def [] filename, read_shebang = false
+			name = File.basename filename
+			ext = File.extname name
+			ext.sub!(/^\./, '')  # delete the leading dot
+			
+			type = 
+				TypeFromExt[ext] || 
+				TypeFromExt[ext.downcase] || 
+				TypeFromName[name] ||
+				TypeFromName[name.downcase]
+			type ||= shebang(filename) if read_shebang
+
+			type
+		end
+
+		def shebang filename
+			begin
+				File.open filename, 'r' do |f|
+					first_line = f.gets
+					first_line[TypeFromShebang]
+				end
+			rescue IOError
+				nil
+			end
+		end
+		
+		# This works like Hash#fetch.
+		def fetch filename, default = nil, read_shebang = false
+			if default and block_given?
+				warn 'block supersedes default value argument'
+			end
+			
+			unless type = self[filename, read_shebang]
+				return yield if block_given?
+				return default if default
+				raise UnknownFileType, 'Could not determine type of %p.' % filename
+			end
+			type
+		end
+
+	end
+
+	TypeFromExt = {
+		'rb' => :ruby,
+		'rbw' => :ruby,
+		'cpp' => :cpp,
+		'c' => :c,
+		'h' => :c,
+		'xml' => :xml,
+		'htm' => :html,
+		'html' => :html,
+	}
+
+	TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
+
+	TypeFromName = {
+		'Rakefile' => :ruby,
+		'Rantfile' => :ruby,
+	}
+
+end
+
+if $0 == __FILE__
+	$VERBOSE = true
+  eval DATA.read, nil, $0, __LINE__+4
+end
+
+__END__
+
+require 'test/unit'
+
+class TC_FileType < Test::Unit::TestCase
+
+	def test_fetch
+		assert_raise FileType::UnknownFileType do
+			FileType.fetch ''
+		end
+
+		assert_throws :not_found do
+			FileType.fetch '.' do
+				throw :not_found
+			end
+		end
+
+		assert_equal :default, FileType.fetch('c', :default)
+		
+		stderr, fake_stderr = $stderr, Object.new
+		$err = ''
+		def fake_stderr.write x
+			$err << x
+		end
+		$stderr = fake_stderr
+		FileType.fetch('c', :default) { }
+		assert_equal "block supersedes default value argument\n", $err
+		$stderr = stderr
+	end
+
+	def test_ruby
+		assert_equal :ruby, FileType['test.rb']
+		assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
+		assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
+		assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
+		assert_not_equal :ruby, FileType['test_rb']
+		assert_not_equal :ruby, FileType['Makefile']
+		assert_not_equal :ruby, FileType['set.rb/set']
+		assert_not_equal :ruby, FileType['~/projects/blabla/rb']
+	end
+
+	def test_c
+		assert_equal :c, FileType['test.c']
+		assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
+		assert_not_equal :c, FileType['test_c']
+		assert_not_equal :c, FileType['Makefile']
+		assert_not_equal :c, FileType['set.h/set']
+		assert_not_equal :c, FileType['~/projects/blabla/c']
+	end
+
+	def test_shebang
+		dir = './test'
+		if File.directory? dir
+			Dir.chdir dir do
+				assert_equal :c, FileType['test.c']
+			end
+		end
+	end
+
+end