coderay 0.5.0.121 → 0.7.1.147

data/FOLDERS

@@ -0,0 +1,53 @@
+= CodeRay - Trunk folder structure
+
+== bench - Benchmarking system
+
+All benchmarking stuff goes here.
+
+Test inputs are stored in files named <code>example.<lang></code>.
+Test outputs go to <code>bench/test.<encoder-default-file-extension></code>.
+
+Run <code>bench/bench.rb</code> to get a usage description.
+
+Run <code>rake bench</code> to perform an example benchmark.
+
+
+== bin - Scripts
+
+Executional files for CodeRay.
+
+
+== demo - Demos and functional tests
+
+Demonstrational scripts to show of CodeRay's features.
+
+Run them as functional tests with <code>rake test:demos</code>.
+
+
+== etc - Lots of stuff
+
+Some addidtional files for CodeRay, mainly graphics and Vim scripts.
+
+
+== gem_server - Gem output folder
+
+For <code>rake gem</code>.
+
+
+== lib - CodeRay library code
+
+This is the base directory for the CodeRay library.
+
+
+== rake_helpers - Rake helper libraries
+
+Some files to enhance Rake, including the Autumnal Rdoc template and some scripts.
+
+
+== test - Tests
+
+Test for the scanners.
+
+Each language has its own subfolder and sub-suite.
+
+Run with <code>rake test</code>.

data/README

@@ -11,20 +11,21 @@ Keywords, strings, floats, comments - all in different colors.
 And with line numbers.
 
 *Syntax* *Highlighting*...
-* makes code easier to read
-* lets you detect errors faster
+* makes code easier to read and maintain
+* lets you detect syntax errors faster
 * helps you to understand the syntax of a language
 * looks nice
 * is what everybody should have on their website
 * solves all your problems and makes the girls run after you
 
-Version: 0.5.0 (2006.march.16)
-Author:: murphy
-Idea:: licenser
-Website:: rd.cYcnus.de/coderay[http://rd.cYcnus.de/coderay]
+Version: 0.5.0 (2006.april.16)
+Author:: murphy (Kornelius Kalnbach)
+Contact:: murphy rubychan de
+Website:: coderay.rubychan.de[http://coderay.rubychan.de]
+Old Website:: rd.cYcnus.de/coderay[http://rd.cYcnus.de/coderay]
 Copyright:: (c) 2006 by cYcnus
 License:: GNU LGPL; see LICENSE file in the main directory.
-Subversion:: $Id: README 119 2006-03-16 00:43:02Z murphy $
+Subversion:: $Id: README 139 2006-04-16 04:31:10Z murphy $
 
 -----
 
@@ -34,10 +35,15 @@ You need RubyGems[http://rubyforge.org/frs/?group_id=126].
 
  % gem install coderay
 
+Since CodeRay is still in beta stage, nightly buildy may be useful:
+
+ % gem install coderay -rs rd.cYcnus.de/coderay
+
 
 === Dependencies
 
-CodeRay needs Ruby 1.8 and the strscan[http://www.ruby-doc.org/stdlib/libdoc/strscan/rdoc/index.htm] library included.
+CodeRay needs Ruby 1.8 and the strscan[http://www.ruby-doc.org/stdlib/libdoc/strscan/rdoc/index.htm] library
+(part of the standard library.)
 
 
 == Example Usage
@@ -45,8 +51,9 @@ CodeRay needs Ruby 1.8 and the strscan[http://www.ruby-doc.org/stdlib/libdoc/str
 
  require 'coderay'
  
- hl = CodeRay.html :line_numbers => :table
- puts hl.highlight_page "puts 'Hello, world!'", :ruby
+ tokens = CodeRay.scan "puts 'Hello, world!'", :ruby
+ page = tokens.html :line_numbers => :inline, :wrap => :page
+ puts page
 
 
 == Documentation
@@ -74,9 +81,10 @@ Please report errors in this documentation to <coderay cycnus de>.
 * Doug Kearns and everyone else who worked on ruby.vim - it not only helped me coding CodeRay,
   but also gave me a wonderful target to reach for the Ruby scanner.
 * everyone who used CodeRay on http://www.rubyforen.de and http://www.infhu.de/mx
-* iGEL, magichisoka, manveru and everyone I forgot from rubyforen.de
+* iGEL, magichisoka, manveru, WoN�Do and everyone I forgot from rubyforen.de
+* Daniel and Dethix from ruby-mine.de
 * Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
-* matz, nobu, why, dave, dhh, etc... you know, those Ruby gods and gurus
+* matz and all Ruby gods and gurus
 * The inventors of: the computer, the internet, the true color display, HTML & CSS, VIM, RUBY,
   pizza, microwaves, guitars, scouting, programming, anime, manga, coke and green ice tea.
 
@@ -94,7 +102,7 @@ Where would we be without all those people?
 * RubyGems[http://docs.rubygems.org/]
 * {Subversion/TortoiseSVN}[http://tortoisesvn.tigris.org/] using Apache via XAMPP[http://www.apachefriends.org/en/xampp.html]
 * RDoc (though I'm quite unsatisfied with it)
-* GNUWin32, MinGW and some other tools to make the shell under windows a bit more usefull
+* GNUWin32, MinGW and some other tools to make the shell under windows a bit more useful
 
 ---
 

data/bin/coderay

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# CodeRay Executable
+# 
+# Version: 0.1
+# Author: murphy
+
+def err msg
+	$stderr.puts msg
+end
+
+begin
+	require 'coderay'
+
+	if ARGV.empty?
+		puts <<-USAGE
+CodeRay #{CodeRay::Version} (http://rd.cYcnus.de/coderay)
+Usage:
+  coderay -<lang> [-<format>] < file > output
+  coderay file [-<format>]
+Example:
+  coderay -ruby -statistic < foo.rb
+  coderay codegen.c  # generates codegen.c.html
+		USAGE
+	end
+	
+	first, second = ARGV
+
+	if first
+		if first[/-(\w+)/] == first
+			lang = $1.to_sym
+			input = $stdin.read
+			tokens = CodeRay.scan input, lang
+		elsif first == '-'
+			lang = $1.to_sym
+			input = $stdin.read
+			tokens = CodeRay.scan input, lang
+		else
+			file = first
+			tokens = CodeRay.scan_file file
+			output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
+		end
+	else
+		puts 'No lang/file given.'
+		exit 1
+	end
+
+	if second
+		if second[/-(\w+)/] == second
+			format = $1.to_sym
+		else
+			raise 'Invalid format (must be -xxx).'
+		end
+	else
+		$stderr.puts 'No format given; setting to default (HTML Page)'
+		format = :page
+	end
+
+	output = tokens.encode format
+	out = $stdout
+	if output_filename
+		output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
+		if File.exist? output_filename
+			err 'File %s already exists.' % output_filename
+			exit
+		else
+			out = File.open output_filename, 'w'
+		end
+	else
+		
+	end
+	out.print output
+
+rescue => boom
+	err "Error: #{boom.message}\n"
+	err boom.backtrace
+	err '-' * 50
+	err ARGV.options
+	exit 1
+end

data/demo/demo_cache.rb

@@ -0,0 +1,12 @@
+require 'coderay'
+
+html_encoder = CodeRay.encoder :html
+
+scanner = Hash.new do |h, lang|
+	h[lang] = CodeRay.scanner lang
+end
+
+for lang in [:ruby, :html]
+	tokens = scanner[lang].tokenize 'test <test>'
+	puts html_encoder.encode_tokens(tokens)
+end

data/demo/demo_html_list.rb

@@ -0,0 +1,12 @@
+$: << '..'
+require 'coderay'
+
+tokens = CodeRay.scan File.read(__FILE__), :ruby
+html = tokens.html(:tab_width => 2, :line_numbers => :inline, :line_number_start => -1)
+
+puts html.page
+
+commment = <<_
+This code must be > 10 lines
+because I want to test the correct adjustment of the line numbers.
+_

data/lib/coderay.rb

@@ -1,6 +1,6 @@
 # = CodeRay Library
 #
-# $Id: coderay.rb 120 2006-03-16 18:35:43Z murphy $
+# $Id: coderay.rb 141 2006-05-11 23:32:49Z murphy $
 #
 # CodeRay is a Ruby library for syntax highlighting.
 #
@@ -133,11 +133,12 @@ module CodeRay
 	# Minor: odd for beta, even for stable
 	# Teeny: development state
 	# Revision: Subversion Revision number (generated on rake)
-	Version = '0.6.0'
+	Version = '0.7.1'
 	
 	require 'coderay/tokens'
 	require 'coderay/scanner'
 	require 'coderay/encoder'
+	require 'coderay/duo'
 	require 'coderay/style'
 
 
@@ -271,6 +272,14 @@ module CodeRay
 			Encoders[format].new options
 		end
 
+		# Finds the Scanner class for +lang+ and creates an instance, passing
+		# +options+ to it.
+		#
+		# See Scanner.new.
+		def scanner lang, options = {}
+			Scanners[lang].new '', options
+		end
+
 		# Extract the options for the scanner from the +options+ hash.
 		#
 		#	Returns an empty Hash if <tt>:scanner_options</tt> is not set.

data/lib/coderay/duo.rb

@@ -0,0 +1,29 @@
+module CodeRay
+	
+	# = Duo
+	# 
+	# $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $
+	# 
+	# TODO: Doc.
+	class Duo
+		
+		attr_accessor :scanner, :encoder
+		
+		def initialize lang, format, options = {}
+			@scanner = CodeRay.scanner lang, CodeRay.get_scanner_options(options)
+			@encoder = CodeRay.encoder format, options
+		end
+
+		class << self
+			alias [] new
+		end
+
+		def encode code
+			@scanner.string = code
+			@encoder.encode_tokens(scanner.tokenize)
+		end
+		alias highlight encode
+		
+	end
+	
+end

data/lib/coderay/encoder.rb

@@ -130,10 +130,12 @@ module CodeRay
 			# By default, it calls text_token or block_token, depending on
 			# whether +text+ is a String.
 			def token text, kind
-				if text.is_a? String
+				if text.is_a? ::String
 					text_token text, kind
-				else
+				elsif text.is_a? ::Symbol
 					block_token text, kind
+				else
+					raise 'Unknown token text type: %p' % text
 				end
 			end
 
@@ -169,5 +171,3 @@ module CodeRay
 
 	end
 end
-
-# vim:sw=2:ts=2:noet:tw=78

data/lib/coderay/encoders/_map.rb

@@ -1,8 +1,8 @@
 module CodeRay
-	module Encoders
-		
-		map :stats => :statistic,
-			:plain => :text
-		
-	end
+module Encoders
+	
+	map :stats => :statistic,
+		:plain => :text
+	
+end
 end

data/lib/coderay/encoders/count.rb

@@ -6,12 +6,12 @@ module Encoders
 		include Streamable
 		register_for :count
 
-	protected		
-		
+		protected		
+
 		def setup options
 			@out = 0
 		end
-		
+
 		def token text, kind
 			@out += 1
 		end

data/lib/coderay/encoders/debug.rb

@@ -1,38 +1,46 @@
 module CodeRay
-	module Encoders
-
-		# = Debug Encoder
-		class Debug < Encoder
-
-			include Streamable
-			register_for :debug
-
-			FILE_EXTENSION = 'raydebug'
-
-		protected
-			def text_token text, kind
-				@out <<
-					if kind == :space
-						text
-					else
-						text = text.gsub(/[)\\]/, '\\\\\0')
-						"#{kind}(#{text})"
-					end					
-			end
-
-			def block_token action, kind
-				@out << super
-			end
+module Encoders
+
+	# = Debug Encoder
+	#
+	# Fast encoder producing simple debug output.
+	#
+	# It is readable and diff-able and is used for testing.
+	#
+	# You cannot fully restore the tokens information from the
+	# output, because consecutive :space tokens are merged.
+	# Use Tokens#dump for caching purposes.
+	class Debug < Encoder
+
+		include Streamable
+		register_for :debug
+
+		FILE_EXTENSION = 'raydebug'
+
+	protected
+		def text_token text, kind
+			@out <<
+				if kind == :space
+					text
+				else
+					text = text.gsub(/[)\\]/, '\\\\\0')
+					"#{kind}(#{text})"
+				end					
+		end
 
-			def open_token kind
-				"#{kind}<"
-			end
+		def block_token action, kind
+			@out << super
+		end
 
-			def close_token kind
-				">"
-			end
+		def open_token kind
+			"#{kind}<"
+		end
 
+		def close_token kind
+			">"
 		end
 
 	end
+
+end
 end

data/lib/coderay/encoders/div.rb

@@ -1,4 +1,5 @@
-module CodeRay module Encoders
+module CodeRay
+module Encoders
 	
 	load :html
 
@@ -15,4 +16,5 @@ module CodeRay module Encoders
 
 	end
 
-end end
+end
+end