RubyGems - coderay - Versions diffs - 0.4.3.48 → 0.4.5.73 - Mend

coderay 0.4.3.48 → 0.4.5.73

Files changed (17) hide show

data/demo/demo_highlight.rb +14 -0
data/demo/demo_server.rb +26 -8
data/lib/coderay.rb +23 -3
data/lib/coderay/encoder.rb +26 -4
data/lib/coderay/encoders/debug.rb +38 -0
data/lib/coderay/encoders/helpers/html_css.rb +1 -0
data/lib/coderay/encoders/helpers/html_helper.rb +1 -0
data/lib/coderay/encoders/helpers/html_output.rb +32 -11
data/lib/coderay/encoders/html.rb +86 -16
data/lib/coderay/encoders/null.rb +9 -3
data/lib/coderay/encoders/statistic.rb +15 -10
data/lib/coderay/encoders/xml.rb +70 -0
data/lib/coderay/helpers/plugin.rb +6 -6
data/lib/coderay/scanners/helpers/ruby_helper.rb +18 -17
data/lib/coderay/scanners/ruby.rb +89 -41
data/lib/coderay/tokens.rb +3 -4
metadata +21 -18

data/demo/demo_highlight.rb ADDED Viewed

@@ -0,0 +1,14 @@
+require 'coderay'
+puts CodeRay.highlight('puts "Hello, World!"', :ruby)
+output = CodeRay.highlight_file($0, :line_numbers => :table)
+puts <<HTML
+<html>
+<head>
+#{output.stylesheet true}
+<body>
+#{output}
+</body>
+</html>
+HTML

data/demo/demo_server.rb CHANGED Viewed

@@ -1,10 +1,24 @@
 # CodeRay dynamic highlighter
-#
-# Usage: start this and your browser.
-#
-# Go to http://localhost:49374/?<path to the file>
-# (mnemonic: 49374 = Four-Nine-Three-Seven-Four = For No Token Shall Fall)
-# and you should get the highlighted version.
+unless ARGV.grep(/-[hv]|--(help|version)/).empty?
+	puts <<-USAGE
+CodeRay Server 0.5
+$Id: demo_server.rb 57 2005-10-22 01:16:34Z murphy $
+Usage:
+	1) Start this and your browser.
+	2) Go to http://localhost:2468/?<path to the file>
+	   and you should get the highlighted version.
+Parameters:
+	-d     Debug mode; reload CodeRay engine for every file.
+	       (prepare for MANY "already initialized" and "method redefined"
+	       messages - ingore it.)
+	...    More to come.
+	USAGE
+	exit
+end
 require 'webrick'
 require 'pathname'
@@ -29,7 +43,7 @@ require 'coderay'
 class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 	STYLE = 'style="font-family: sans-serif; color: navy;"'
-	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="HIghlighted by CodeRay"/></p>'
+	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="Highlighted by CodeRay"/></p>'
 	def do_GET req, res
 		q = req.query_string || ''
@@ -59,6 +73,10 @@ class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 				page << "#{BANNER}</body></html>"
 			elsif File.exist? path
+				if $DEBUG
+					$".delete_if { |f| f =~ /coderay/ }
+					require 'coderay'
+				end
 				div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div
 				div.replace <<-DIV
 	<div #{STYLE}>
@@ -74,7 +92,7 @@ class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 	end
 end
-# this is taken by "qip_msgd" - I don't know that.
+# This port is taken by "qip_msgd" - I don't know that. Do you?
 module CodeRay
 	PORT = 0xC0DE / 20
 end

data/lib/coderay.rb CHANGED Viewed

@@ -1,6 +1,6 @@
 # = CodeRay Library
 #
-# $Id: coderay.rb 44 2005-10-01 00:59:05Z murphy $
+# $Id: coderay.rb 73 2005-10-29 16:09:17Z murphy $
 #
 # CodeRay is a Ruby library for syntax highlighting.
 #
@@ -128,7 +128,7 @@
 #	The scanning methods provide more flexibility; we recommend to use these.
 module CodeRay
-	Version = '0.4.4'
+	Version = '0.4.6'
 	require 'coderay/tokens'
 	require 'coderay/scanner'
@@ -201,6 +201,16 @@ module CodeRay
 			encoder(format, options).encode code, lang, options
 		end
+		# Highlight a string into a HTML <div>.
+		#
+		# CSS styles use classes, so you have to include a stylesheet
+		# in your output.
+		#
+		# See encode.
+		def highlight code, lang, options = { :css => :class }, format = :div
+			encode code, lang, format, options
+		end
 		# Encode pre-scanned Tokens.
 		# Use this together with CodeRay.scan:
 		#
@@ -223,10 +233,20 @@ module CodeRay
 		#  require 'coderay'
 		#  page = CodeRay.encode_file 'some_c_code.c', :html
 		def encode_file filename, format, options = {}
-			tokens = scan_file filename, auto, get_scanner_options(options)
+			tokens = scan_file filename, :auto, get_scanner_options(options)
 			encode_tokens tokens, format, options
 		end
+		# Highlight a file into a HTML <div>.
+		#
+		# CSS styles use classes, so you have to include a stylesheet
+		# in your output.
+		#
+		# See encode.
+		def highlight_file filename, options = { :css => :class }, format = :div
+			encode_file filename, format, options
+		end
 		# Finds the Encoder class for +format+ and creates an instance, passing
 		# +options+ to it.
 		#

data/lib/coderay/encoder.rb CHANGED Viewed

@@ -109,6 +109,11 @@ module CodeRay
 				method(:token).to_proc
 			end
+			# Return the default file extension for outputs of this encoder.
+			def file_extension
+				self.class::FILE_EXTENSION
+			end
 		protected
 			# Called with merged options before encoding starts.
@@ -122,11 +127,28 @@ module CodeRay
 			# Called with +text+ and +kind+ of the currently scanned token.
 			# For simple scanners, it's enougth to implement this method.
 			#
-			# Raises a NotImplementedError exception if it is not overwritten
-			# in subclass.
+			# By default, it calls text_token or block_token, depending on
+			# whether +text+ is a String.
 			def token text, kind
-				raise NotImplementedError,
-					"#{self.class}#token not implemented."
+				if text.is_a? String
+					text_token text, kind
+				else
+					block_token text, kind
+				end
+			end
+			def text_token text, kind
+			end
+			def block_token action, kind
+				case action
+				when :open
+					open_token kind
+				when :close
+					close_token kind
+				else
+					raise 'unknown block action: %p' % action
+				end
 			end
 			# Called with merged options after encoding starts.

data/lib/coderay/encoders/debug.rb ADDED Viewed

@@ -0,0 +1,38 @@
+module CodeRay
+	module Encoders
+		# = Debug Encoder
+		class Debug < Encoder
+			include Streamable
+			register_for :debug
+			FILE_EXTENSION = 'raydebug'
+		protected
+			def text_token text, kind
+				@out <<
+					if kind == :space
+						text
+					else
+						text = text.gsub(/[)\\]/, '\\\\\0')
+						"#{kind}(#{text})"
+					end
+			end
+			def block_token action, kind
+				@out << super
+			end
+			def open_token kind
+				"#{kind}<"
+			end
+			def close_token kind
+				">"
+			end
+		end
+	end
+end

data/lib/coderay/encoders/helpers/html_css.rb CHANGED Viewed

@@ -102,6 +102,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .hx { color:#058; font-weight:bold; }
 .i  { color:#00D; font-weight:bold; }
 .ic { color:#B44; font-weight:bold; }
+.il { }
 .in { color:#B2B; font-weight:bold; }
 .iv { color:#33B; }
 .la { color:#970; font-weight:bold; }

data/lib/coderay/encoders/helpers/html_helper.rb CHANGED Viewed

@@ -28,6 +28,7 @@ module CodeRay module Encoders
 			:global_variable => 'gv',
 			:hex => 'hx',
 			:include => 'ic',
+			:inline => 'il',
 			:instance_variable => 'iv',
 			:integer => 'i',
 			:interpreted => 'in',

data/lib/coderay/encoders/helpers/html_output.rb CHANGED Viewed

@@ -13,6 +13,8 @@ module CodeRay
 		# TODO: more doc.
 		module Output
+			attr_accessor :wrapped_in
 			class << self
 				# This makes Output look like a class.
@@ -33,8 +35,18 @@ module CodeRay
 					warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
 				end
+				def stylesheet in_tag = false
+					ss = CSS::DEFAULT_STYLESHEET
+					ss = <<-CSS if in_tag
+<style type="text/css">
+#{ss}
+</style>
+					CSS
+					ss
+				end
 				def page_template_for_css css = :default
-					css = CSS::DEFAULT_STYLESHEET if css == :default
+					css = stylesheet if css == :default
 					PAGE.apply 'CSS', css
 				end
@@ -53,11 +65,6 @@ module CodeRay
 			wrapper :div, :span, :page
-			def wrapped_in
-				@wrapped_in ||= nil
-			end
-			attr_writer :wrapped_in
 			def wrapped_in? element
 				wrapped_in == element
 			end
@@ -97,19 +104,28 @@ module CodeRay
 				clone.wrap!(*args)
 			end
+			NUMERIZABLE_WRAPPINGS = {
+				:table => [:div, :page],
+				:inline => :all,
+				nil => :all
+			}
 			def numerize! mode = :table, options = {}
 				return self unless mode
-				start = options.fetch :line_number_start, DEFAULT_OPTIONS[:line_number_start]
+				options = DEFAULT_OPTIONS.merge options
+				start = options[:line_number_start]
 				unless start.is_a? Integer
 					raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
 				end
-				unless NUMERIZABLE_WRAPPINGS.include? options[:wrap]
+				allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
+				unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
 					raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
 				end
-				bold_every = options.fetch :bold_every, DEFAULT_OPTIONS[:bold_every]
+				bold_every = options[:bold_every]
 				bolding =
 					if bold_every == :no_bolding or bold_every == 0
 						proc { |line| line.to_s }
@@ -137,7 +153,7 @@ module CodeRay
 						line += 1
 						"<span class=\"no\">#{ line_number.rjust(max_width) }</span>  "
 					end
-					wrap! :div
+					#wrap! :div
 				when :table
 					# This is really ugly.
@@ -164,6 +180,10 @@ module CodeRay
 				clone.numerize!(*args)
 			end
+			def stylesheet in_tag = false
+				Output.stylesheet in_tag
+			end
 			class Template < String
 				def self.wrap! str, template, target
@@ -207,9 +227,10 @@ module CodeRay
 			DIV_TABLE = <<-`DIV_TABLE`
 <table class="CodeRay"> <tr>
 	<td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
-	<td class="code"><pre title="double click to expand" ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
+	<td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
 </tr> </table>
 			DIV_TABLE
+			# title="double click to expand"
 			PAGE = <<-`PAGE`
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

data/lib/coderay/encoders/html.rb CHANGED Viewed

@@ -1,6 +1,63 @@
 module CodeRay
 module Encoders
+	# = HTML Encoder
+	#
+	# This is CodeRay's most important highlighter:
+	# It provides save, fast XHTML generation and CSS support.
+	#
+	# == Usage
+	#
+	#  require 'coderay'
+	#  puts CodeRay.scan('Some /code/', :ruby).html  #-> a HTML page
+	#  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span) #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+	#  puts CodeRay.scan('Some /code/', :ruby).span  #-> the same
+	#
+	#  puts CodeRay.scan('Some code', :ruby).html(
+	#    :wrap => nil,
+	#    :line_numbers => :inline,
+	#    :css => :style
+	#  )
+	#  #-> <span class="no">1</span>  <span style="color:#036; font-weight:bold;">Some</span> code
+	#
+	# == Options
+	#
+	# === :tab_width
+	# Convert \t characters to +n+ spaces (a number.)
+	# Default: 8
+	#
+	# === :css
+	# How to include the styles; can be :class or :style.
+	#
+	# Default: :class
+	#
+	# === :wrap
+	# Wrap in :page, :div, :span or nil.
+	#
+	# You can also use Encoders::Div and Encoders::Span.
+	#
+	# Default: :page
+	#
+	# === :line_numbers
+	# Include line numbers in :table, :inline or nil (no line numbers)
+	#
+	# Default: nil
+	#
+	# === :line_number_start
+	# Where to start with line number counting.
+	#
+	# Default: 1
+	#
+	# === :bold_every
+	# Make every +n+-th number appear bold.
+	#
+	# Default: 10
+	#
+	# === :hint
+	# Include some information into the output using the title attribute.
+	# Can be :info (show token type on mouse-over) or :debug.
+	#
+	# Default: false
 	class HTML < Encoder
 		include Streamable
@@ -19,8 +76,9 @@ module Encoders
 			:line_numbers => nil,
 			:line_number_start => 1,
 			:bold_every => 10,
+			:hint => false,
 		}
-		NUMERIZABLE_WRAPPINGS = [:div, :page]
 		require 'coderay/encoders/helpers/html_helper'
 		require 'coderay/encoders/helpers/html_output'
@@ -47,12 +105,9 @@ module Encoders
 		ansi_chars = Array(0x7f..0xff)
 		ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
 		# \x9 (\t) and \xA (\n) not included
-		HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/
+		HTML_ESCAPE_PATTERN = /[\t&"><\xB-\x1f\x7f-\xff\0-\x8]/
 		def setup options
-			if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap]
-				warn ':line_numbers wanted, but :wrap is %p' % options[:wrap]
-			end
 			super
 			return if options == @last_options
 			@last_options = options
@@ -63,6 +118,11 @@ module Encoders
 			@opened = [nil]
 			@css = CSS.new
+			hint = options[:hint]
+			if hint and not [:debug, :info].include? hint
+				raise ArgumentError, "Unknown value %p for :hint; expected :info, :debug, false or nil." % hint
+			end
 			case options[:css]
 			when :class
@@ -73,15 +133,20 @@ module Encoders
 						type = k
 					end
 					c = ClassOfKind[type]
-					if c == :NO_HIGHLIGHT
+					if c == :NO_HIGHLIGHT and not hint
 						h[k] = false
 					else
-						if options[:debug]
-							debug_info = ' title="%p"' % [ k ]
+						title = if hint
+							if hint == :debug
+								' title="%p"' % [ k ]
+							elsif hint == :info
+								path = (k[1..-1] << k.first).map { |kind| kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } }
+								" title=\"#{path.join('/')}\""
+							end
 						else
-							debug_info = ''
+							''
 						end
-						h[k] = '<span%s class="%s">' % [debug_info, c]
+						h[k] = '<span%s class="%s">' % [title, c]
 					end
 				end
@@ -92,19 +157,24 @@ module Encoders
 					else
 						styles = [k]
 					end
+					type = styles.first
 					styles.map! { |c| ClassOfKind[c] }
-					if styles.first == :NO_HIGHLIGHT
+					if styles.first == :NO_HIGHLIGHT and not hint
 						h[k] = false
 					else
-						if options[:debug]
-							debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ]
+						title = if hint
+							if hint == :debug
+								' title="%p"' % [ styles ]
+							elsif hint == :info and styles.size == 1
+								" title=\"#{type.to_s.gsub(/_/, " ").capitalize}\""
+							end
 						else
-							debug_info = ''
+							''
 						end
 						style = @css[*styles]
 						h[k] =
 							if style
-								'<span%s style="%s">' % [debug_info, style]
+								'<span%s style="%s">' % [title, style]
 							else
 								false
 							end
@@ -112,7 +182,7 @@ module Encoders
 				end
 			else
-				raise "Unknown value %p for :css." % options[:css]
+				raise ArgumentError, "Unknown value %p for :css." % options[:css]
 			end
 		end

data/lib/coderay/encoders/null.rb CHANGED Viewed

@@ -1,12 +1,20 @@
 module CodeRay
 	module Encoders
+		# = Null Encoder
+		#
+		# Does nothing and returns an empty string.
 		class Null < Encoder
 			include Streamable
 			register_for :null
-			protected
+			# Defined for faster processing
+			def to_proc
+				proc {}
+			end
+		protected
 			def token(*)
 				# do nothing
@@ -16,5 +24,3 @@ module CodeRay
 	end
 end

data/lib/coderay/encoders/statistic.rb CHANGED Viewed

@@ -22,17 +22,22 @@ module CodeRay module Encoders
 			super
 		end
-		def token text, type
+		def text_token text, kind
+			@real_token_count += 1 unless kind == :space
+			@type_stats[kind].count += 1
+			@type_stats[kind].size += text.size
+			@type_stats['TOTAL'].size += text.size
+		end
+		# TODO Hierarchy handling
+		def block_token action, kind
+			#@content_type = kind
+			@type_stats['open/close'].count += 1
+		end
+		def token text, kind
+			super
 			@type_stats['TOTAL'].count += 1
-			if text.is_a? String
-				@real_token_count += 1 unless type == :space
-				@type_stats[type].count += 1
-				@type_stats[type].size += text.size
-				@type_stats['TOTAL'].size += text.size
-			else
-				@content_type = type
-				@type_stats['open/close'].count += 1
-			end
 		end
 		STATS = <<-STATS

data/lib/coderay/encoders/xml.rb ADDED Viewed

@@ -0,0 +1,70 @@
+module CodeRay
+	module Encoders
+		# = Debug Encoder
+		class XML < Encoder
+			include Streamable
+			register_for :xml
+			FILE_EXTENSION = 'xml'
+			require 'rexml/document'
+			DEFAULT_OPTIONS = {
+				:tab_width => 8,
+				:pretty => -1,
+				:transitive => false,
+			}
+		protected
+			def setup options
+				@out = ''
+				@doc = REXML::Document.new
+				@doc << REXML::XMLDecl.new
+				@tab_width = options[:tab_width]
+				@root = @node = @doc.add_element('coderay-tokens')
+			end
+			def finish options
+				@doc.write @out, options[:pretty], options[:transitive], true
+				@out
+			end
+			def text_token text, kind
+				if kind == :space
+					token = @node
+				else
+					token = @node.add_element kind.to_s
+				end
+				text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+					case
+					when space
+						token << REXML::Text.new(space, true)
+					when tab
+						token << REXML::Text.new(tab, true)
+					when nl
+						token << REXML::Text.new(nl, true)
+					else
+						token << REXML::Text.new($&)
+					end
+				end
+			end
+			def open_token kind
+				@node = @node.add_element kind.to_s
+			end
+			def close_token kind
+				if @node == @root
+					raise 'no token to close!'
+				end
+				@node = @node.parent
+			end
+		end
+	end
+end

data/lib/coderay/helpers/plugin.rb CHANGED Viewed

@@ -1,6 +1,6 @@
 # = PluginHost
 #
-# $Id: plugin.rb 47 2005-10-01 06:04:52Z murphy $
+# $Id: plugin.rb 59 2005-10-29 04:55:15Z murphy $
 #
 # A simple subclass plugin system.
 #
@@ -90,10 +90,10 @@ module PluginHost
 				id = validate_id(plugin_id)
 				path = path_to id
 				begin
-					$stderr.puts 'Loading plugin: ' + path if $DEBUG
+					#$stderr.puts 'Loading plugin: ' + path if $DEBUG
 					require path
-				rescue LoadError
-					raise PluginNotFound, "Plugin #{id.inspect} not found."
+				rescue LoadError => boom
+					raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
 				else
 					# Plugin should have registered by now
 					unless h.has_key? id
@@ -151,7 +151,7 @@ module PluginHost
 	end
 	# Alias for +[]+.
-	alias load_plugin []
+	alias load []
 	# Returns the Plugin for +id+.
 	# Use it like Hash#fetch.
@@ -239,7 +239,7 @@ def require_plugin path
 	host = PluginHost.host_by_id(host_id)
 	raise PluginHost::HostNotFound,
 		"No host for #{host_id.inspect} found." unless host
-	host.load_plugin plugin_id
+	host.load plugin_id
 end

data/lib/coderay/scanners/helpers/ruby_helper.rb CHANGED Viewed

@@ -12,9 +12,11 @@ module CodeRay module Scanners
 		]
 		DEF_KEYWORDS = %w[ def ]
+		UNDEF_KEYWORDS = %w[ undef ]
 		MODULE_KEYWORDS = %w[class module]
 		DEF_NEW_STATE = WordList.new(:initial).
 			add(DEF_KEYWORDS, :def_expected).
+			add(UNDEF_KEYWORDS, :undef_expected).
 			add(MODULE_KEYWORDS, :module_expected)
 		IDENTS_ALLOWING_REGEXP = %w[
@@ -32,13 +34,11 @@ module CodeRay module Scanners
 			add(RESERVED_WORDS, :reserved).
 			add(PREDEFINED_CONSTANTS, :pre_constant)
-#		IDENT = /[a-zA-Z_][a-zA-Z_0-9]*/
 		IDENT = /[a-z_][\w_]*/i
 		METHOD_NAME = / #{IDENT} [?!]? /ox
-		METHOD_NAME_EX = /
-			#{IDENT}[?!=]?  # common methods: split, foo=, empty?, gsub!
-			| \*\*?         # multiplication and power
+		METHOD_NAME_OPERATOR = /
+			\*\*?           # multiplication and power
 			| [-+]@?        # plus, minus
 			| [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
 			| \[\]=?        # array getter and setter
@@ -46,10 +46,11 @@ module CodeRay module Scanners
 			| <=?>? | >=?   # comparison, rocket operator
 			| ===?          # simple equality and case equality
 		/ox
+		METHOD_NAME_EX = / #{IDENT} [?!=]? | #{METHOD_NAME_OPERATOR} /ox
 		INSTANCE_VARIABLE = / @ #{IDENT} /ox
 		CLASS_VARIABLE = / @@ #{IDENT} /ox
 		OBJECT_VARIABLE = / @@? #{IDENT} /ox
-		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9] | 0[a-zA-Z_0-9]* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
 		PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
 		VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
@@ -60,19 +61,17 @@ module CodeRay module Scanners
 		QUOTE_TO_TYPE.default = :string
 		REGEXP_MODIFIERS = /[mixounse]*/
-		REGEXP_SYMBOLS = /
-			[|?*+?(){}\[\].^$]
-		/x
+		REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/
-		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+		DECIMAL = /\d+(?:_\d+)*/
 		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
 		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
 		BINARY = /0b[01]+(?:_[01]+)*/
 		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-		FLOAT_OR_INT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? )? /ox
-		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /ox
-		NUMERIC = / #{OCTAL} | #{HEXADECIMAL} | #{BINARY} | #{FLOAT_OR_INT} /ox
+		FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+		FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+		NUMERIC = / (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} /ox
 		SYMBOL = /
 			:
@@ -103,7 +102,7 @@ module CodeRay module Scanners
 			)
 		/mx
-		# NOTE: This is not completel correct, but
+		# NOTE: This is not completely correct, but
 		# nobody needs heredoc delimiters ending with \n.
 		HEREDOC_OPEN = /
 			<< (-)?              # $1 = float
@@ -115,7 +114,7 @@ module CodeRay module Scanners
 			)
 		/mx
-		RDOC = /
+		RUBYDOC = /
 			=begin (?!\S)
 			.*?
 			(?: \Z | ^=end (?!\S) [^\n]* )
@@ -127,6 +126,8 @@ module CodeRay module Scanners
 			(?: \Z | (?=^\#CODE) )
 		/mx
+		RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
 		RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
 		FANCY_START = / % ( [qQwWxsr] | (?![\w\s=]) ) (.) /mox
@@ -144,7 +145,7 @@ module CodeRay module Scanners
 		FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
 		class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
-			:paren, :paren_depth, :pattern
+			:paren, :paren_depth, :pattern, :next_state
 			CLOSING_PAREN = Hash[ *%w[
 				( )
@@ -186,7 +187,7 @@ module CodeRay module Scanners
 				delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
 				h[k] =
 					if interpreted
-						/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx
+				/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
 					else
 						/ (?= #{delim_pattern}() | \\ ) /mx
 					end
@@ -203,7 +204,7 @@ module CodeRay module Scanners
 				else
 					pattern = STRING_PATTERN[ [delim, interpreted] ]
 				end
-				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern
+				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
 			end
 		end unless defined? StringState

data/lib/coderay/scanners/ruby.rb CHANGED Viewed

@@ -43,7 +43,6 @@ module CodeRay module Scanners
 				if state.instance_of? StringState
 # {{{
 					match = scan_until(state.pattern) || scan_until(/\z/)
 					tokens << [match, :content] unless match.empty?
 					break if eos?
@@ -52,7 +51,7 @@ module CodeRay module Scanners
 						match = getch + scan_until(/$/)
 						tokens << [match, :delimiter]
 						tokens << [:close, state.type]
-						state = :initial
+						state = state.next_state
 						next
 					end
@@ -103,7 +102,7 @@ module CodeRay module Scanners
 						end
 						tokens << [:close, state.type]
 						fancy_allowed = regexp_allowed = false
-						state = :initial
+						state = state.next_state
 					when '\\'
 						if state.interpreted
@@ -128,13 +127,14 @@ module CodeRay module Scanners
 							fancy_allowed = regexp_allowed = true
 							state = :initial
 							depth = 1
-							tokens << [match + getch, :escape]
+							tokens << [:open, :inline]
+							tokens << [match + getch, :delimiter]
 						when ?$, ?@
 							tokens << [match, :escape]
 							last_state = state  # scan one token as normal code, then return here
 							state = :initial
 						else
-							raise "else-case # reached; #%p not handled" % peek(1), tokens
+							raise 'else-case # reached; #%p not handled' % peek(1), tokens
 						end
 					when state.paren
@@ -145,7 +145,7 @@ module CodeRay module Scanners
 						tokens << [match, :function]
 					else
-						raise "else-case \" reached; %p not handled, state = %p" % [match, state], tokens
+						raise 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
 					end
 					next
@@ -153,7 +153,7 @@ module CodeRay module Scanners
 				else
 # {{{
 					if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
-						( bol? and match = scan(/ #{DATA} | #{RDOC} /ox) )
+						( bol? and match = scan(/#{RUBYDOC_OR_DATA}/o) )
 						fancy_allowed = true
 						case m = match[0]
 						when ?\s, ?\t, ?\f
@@ -161,7 +161,10 @@ module CodeRay module Scanners
 							type = :space
 						when ?\n, ?\\
 							type = :space
-							regexp_allowed = m == ?\n
+							if m == ?\n
+								regexp_allowed = true
+								state = :initial if state == :undef_comma_expected
+							end
 							if heredocs
 								unscan  # heredoc scanning needs \n at start
 								state = heredocs.shift
@@ -175,14 +178,32 @@ module CodeRay module Scanners
 							type = :comment
 							regexp_allowed = true
 						else
-							raise "else-case _ reached, because case %p was not handled" % [matched[0].chr], tokens
+							raise 'else-case _ reached, because case %p was not handled' % [matched[0].chr], tokens
 						end
 						tokens << [match, type]
 						next
 					elsif state == :initial
-						if match = scan(/ \.\.?\.? | [-+*=>;,|&!\(\)\[\]~^]+ | [\{\}] | :: /x)
-							if match !~ / [.\)\]\}] \z/x or match =~ /\.\.\.?/
+						# IDENTS #
+						if match = scan(/#{METHOD_NAME}/o)
+							if last_token_dot
+								type = if match[/^[A-Z]/] then :constant else :ident end
+							else
+								type = IDENT_KIND[match]
+								if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]
+									type = :constant
+								elsif type == :reserved
+									state = DEF_NEW_STATE[match]
+								end
+							end
+							## experimental!
+							fancy_allowed = regexp_allowed = :set if REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
+						# OPERATORS #
+						elsif (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)) or
+							(last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))
+							if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
 								regexp_allowed = fancy_allowed = :set
 							end
 							last_token_dot = :set if match == '.' or match == '::'
@@ -195,33 +216,22 @@ module CodeRay module Scanners
 									depth -= 1
 									if depth == 0
 										state, depth, heredocs = states.pop
-										type = :escape
+										tokens << [match, :delimiter]
+										type = :inline
+										match = :close
 									end
 								end
 							end
-						elsif match = scan(/#{METHOD_NAME}/o)
-							if last_token_dot
-								type = if match[/^[A-Z]/] then :constant else :ident end
-							else
-								type = IDENT_KIND[match]
-								if type == :ident and match[/^[A-Z]/]
-									type = :constant
-								elsif type == :reserved
-									state = DEF_NEW_STATE[match]
-								end
-							end
-							fancy_allowed = regexp_allowed = REGEXP_ALLOWED[match]
 						elsif match = scan(/ ['"] /mx)
 							tokens << [:open, :string]
 							type = :delimiter
-							state = StringState.new :string, match != '\'', match  # important for streaming
+							state = StringState.new :string, match == '"', match  # important for streaming
 						elsif match = scan(/#{INSTANCE_VARIABLE}/o)
 							type = :instance_variable
-						elsif regexp_allowed and match = scan(/ \/ /mx)
+						elsif regexp_allowed and match = scan(/\//)
 							tokens << [:open, :regexp]
 							type = :delimiter
 							interpreted = true
@@ -232,15 +242,23 @@ module CodeRay module Scanners
 							end
 						elsif match = scan(/#{NUMERIC}/o)
-							type = if match[/#{FLOAT}/o] then :float else :integer end
+							type = if self[1] then :float else :integer end
-						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)
-							case match[1]
+						elsif match = scan(/#{SYMBOL}/o)
+							case delim = match[1]
 							when ?', ?"
 								tokens << [:open, :symbol]
-								state = StringState.new :symbol, match[1] == ?", match[1,1]
+								tokens << [':', :symbol]
+								match = delim.chr
+								type = :delimiter
+								state = StringState.new :symbol, delim == ?", match
+							else
+								type = :symbol
 							end
-							type = :symbol
+						elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
+							regexp_allowed = fancy_allowed = :set
+							type = :operator
 						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
 							indented = self[1] == '-'
@@ -265,7 +283,7 @@ module CodeRay module Scanners
 						elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
 							type = :integer
-						elsif match = scan(/ [\/%<?:] /x)
+						elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
 							regexp_allowed = fancy_allowed = :set
 							type = :operator
@@ -290,25 +308,55 @@ module CodeRay module Scanners
 						end
 					elsif state == :def_expected
-						if match = scan(/ (?: #{VARIABLE} (?: ::#{IDENT} )* \. )? #{METHOD_NAME_EX} /ox)
+						state = :initial
+						if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)
 							type = :method
 						else
-							match = getch
+							next
+						end
+					elsif state == :undef_expected
+						state = :undef_comma_expected
+						if match = scan(/#{METHOD_NAME_EX}/o)
+							type = :method
+						elsif match = scan(/#{SYMBOL}/o)
+							case delim = match[1]
+							when ?', ?"
+								tokens << [:open, :symbol]
+								tokens << [':', :symbol]
+								match = delim.chr
+								type = :delimiter
+								state = StringState.new :symbol, delim == ?", match
+								state.next_state = :undef_comma_expected
+							else
+								type = :symbol
+							end
+						else
+							state = :initial
+							next
+						end
+					elsif state == :undef_comma_expected
+						if match = scan(/,/)
+							type = :operator
+							state = :undef_expected
+						else
+							state = :initial
+							next
 						end
-						state = :initial
 					elsif state == :module_expected
 						if match = scan(/<</)
 							type = :operator
 						else
+							state = :initial
 							if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
 								type = :class
 							else
-								match = getch
+								next
 							end
 						end
-						state = :initial
 					end
 					regexp_allowed = regexp_allowed == :set
@@ -316,7 +364,7 @@ module CodeRay module Scanners
 					last_token_dot = last_token_dot == :set
 					if $DEBUG
-						raise_inspect 'error token %p in line %d' % [tokens.last, line], tokens if not type or type == :error
+						raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error
 					end
 					tokens << [match, type]

data/lib/coderay/tokens.rb CHANGED Viewed

@@ -94,15 +94,14 @@ module CodeRay
 			false
 		end
-		alias :orig_each :each
 		# Iterates over all tokens.
 		#
 		# If a filter is given, only tokens of that kind are yielded.
 		def each kind_filter = nil, &block
 			unless kind_filter
-				orig_each(&block)
+				super(&block)
 			else
-				orig_each do |text, kind|
+				super do |text, kind|
 					next unless kind == kind_filter
 					yield text, kind
 				end
@@ -115,7 +114,7 @@ module CodeRay
 		# Example:
 		#   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
 		def each_text_token
-			orig_each do |text, kind|
+			each do |text, kind|
 				next unless text.respond_to? :to_str
 				yield text, kind
 			end

metadata CHANGED Viewed

@@ -3,8 +3,8 @@ rubygems_version: 0.8.10
 specification_version: 1
 name: coderay
 version: !ruby/object:Gem::Version
-  version: 0.4.3.48
-date: 2005-10-01
+  version: 0.4.5.73
+date: 2005-10-29
 summary: CodeRay is a fast syntax highlighter engine for many languages.
 require_paths:
   - lib
@@ -31,47 +31,50 @@ authors:
   - murphy
 files:
   - "./lib/coderay.rb"
-  - "./lib/coderay/encoder.rb"
-  - "./lib/coderay/scanner.rb"
   - "./lib/coderay/tokens.rb"
-  - "./lib/coderay/encoders/yaml.rb"
+  - "./lib/coderay/scanner.rb"
+  - "./lib/coderay/encoder.rb"
+  - "./lib/coderay/encoders/xml.rb"
+  - "./lib/coderay/encoders/span.rb"
   - "./lib/coderay/encoders/div.rb"
+  - "./lib/coderay/encoders/yaml.rb"
   - "./lib/coderay/encoders/tokens.rb"
+  - "./lib/coderay/encoders/text.rb"
   - "./lib/coderay/encoders/statistic.rb"
+  - "./lib/coderay/encoders/count.rb"
   - "./lib/coderay/encoders/null.rb"
-  - "./lib/coderay/encoders/span.rb"
+  - "./lib/coderay/encoders/debug.rb"
   - "./lib/coderay/encoders/html.rb"
-  - "./lib/coderay/encoders/text.rb"
-  - "./lib/coderay/encoders/count.rb"
   - "./lib/coderay/encoders/helpers/html_helper.rb"
-  - "./lib/coderay/encoders/helpers/html_output.rb"
   - "./lib/coderay/encoders/helpers/html_css.rb"
-  - "./lib/coderay/helpers/plugin.rb"
+  - "./lib/coderay/encoders/helpers/html_output.rb"
   - "./lib/coderay/helpers/filetype.rb"
-  - "./lib/coderay/helpers/gzip_simple.rb"
+  - "./lib/coderay/helpers/plugin.rb"
   - "./lib/coderay/helpers/scanner_helper.rb"
+  - "./lib/coderay/helpers/gzip_simple.rb"
   - "./lib/coderay/scanners/c.rb"
   - "./lib/coderay/scanners/delphi.rb"
   - "./lib/coderay/scanners/plaintext.rb"
   - "./lib/coderay/scanners/ruby.rb"
   - "./lib/coderay/scanners/helpers/ruby_helper.rb"
   - "./demo/demo_global_vars2.rb"
+  - "./demo/demo_stream2.rb"
+  - "./demo/demo_load_encoder.rb"
   - "./demo/demo_encoder.rb"
   - "./demo/demo_div.rb"
+  - "./demo/demo_html2.rb"
+  - "./demo/demo_css.rb"
   - "./demo/demo_count.rb"
   - "./demo/demo_simple.rb"
-  - "./demo/demo_tokens.rb"
-  - "./demo/demo_css.rb"
-  - "./demo/demo_html.rb"
-  - "./demo/demo_dump.rb"
-  - "./demo/demo_stream2.rb"
-  - "./demo/demo_load_encoder.rb"
-  - "./demo/demo_html2.rb"
   - "./demo/demo_scanner.rb"
   - "./demo/demo_stream.rb"
   - "./demo/demo_global_vars.rb"
   - "./demo/demo_more.rb"
+  - "./demo/demo_tokens.rb"
+  - "./demo/demo_html.rb"
+  - "./demo/demo_dump.rb"
   - "./demo/demo_server.rb"
+  - "./demo/demo_highlight.rb"
   - "./README"
   - "./LICENSE"
 test_files: []