RubyGems - coderay - Versions diffs - 0.4.3.48 → 0.4.5.73 - Mend

coderay 0.4.3.48 → 0.4.5.73

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

data/demo/demo_highlight.rb +14 -0
data/demo/demo_server.rb +26 -8
data/lib/coderay.rb +23 -3
data/lib/coderay/encoder.rb +26 -4
data/lib/coderay/encoders/debug.rb +38 -0
data/lib/coderay/encoders/helpers/html_css.rb +1 -0
data/lib/coderay/encoders/helpers/html_helper.rb +1 -0
data/lib/coderay/encoders/helpers/html_output.rb +32 -11
data/lib/coderay/encoders/html.rb +86 -16
data/lib/coderay/encoders/null.rb +9 -3
data/lib/coderay/encoders/statistic.rb +15 -10
data/lib/coderay/encoders/xml.rb +70 -0
data/lib/coderay/helpers/plugin.rb +6 -6
data/lib/coderay/scanners/helpers/ruby_helper.rb +18 -17
data/lib/coderay/scanners/ruby.rb +89 -41
data/lib/coderay/tokens.rb +3 -4
metadata +21 -18

data/demo/demo_highlight.rb ADDED Viewed

@@ -0,0 +1,14 @@
+require 'coderay'
+puts CodeRay.highlight('puts "Hello, World!"', :ruby)
+output = CodeRay.highlight_file($0, :line_numbers => :table)
+puts <<HTML
+<html>
+<head>
+#{output.stylesheet true}
+<body>
+#{output}
+</body>
+</html>
+HTML

data/demo/demo_server.rb CHANGED Viewed

@@ -1,10 +1,24 @@
 # CodeRay dynamic highlighter
-#
-# Usage: start this and your browser.
-#
-# Go to http://localhost:49374/?<path to the file>
-# (mnemonic: 49374 = Four-Nine-Three-Seven-Four = For No Token Shall Fall)
-# and you should get the highlighted version.
+unless ARGV.grep(/-[hv]|--(help|version)/).empty?
+	puts <<-USAGE
+CodeRay Server 0.5
+$Id: demo_server.rb 57 2005-10-22 01:16:34Z murphy $
+Usage:
+	1) Start this and your browser.
+	2) Go to http://localhost:2468/?<path to the file>
+	   and you should get the highlighted version.
+Parameters:
+	-d     Debug mode; reload CodeRay engine for every file.
+	       (prepare for MANY "already initialized" and "method redefined"
+	       messages - ingore it.)
+	...    More to come.
+	USAGE
+	exit
+end
 require 'webrick'
 require 'pathname'
@@ -29,7 +43,7 @@ require 'coderay'
 class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 	STYLE = 'style="font-family: sans-serif; color: navy;"'
-	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="HIghlighted by CodeRay"/></p>'
+	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="Highlighted by CodeRay"/></p>'
 	def do_GET req, res
 		q = req.query_string || ''
@@ -59,6 +73,10 @@ class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 				page << "#{BANNER}</body></html>"
 			elsif File.exist? path
+				if $DEBUG
+					$".delete_if { |f| f =~ /coderay/ }
+					require 'coderay'
+				end
 				div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div
 				div.replace <<-DIV
 	<div #{STYLE}>
@@ -74,7 +92,7 @@ class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
 	end
 end
-# this is taken by "qip_msgd" - I don't know that.
+# This port is taken by "qip_msgd" - I don't know that. Do you?
 module CodeRay
 	PORT = 0xC0DE / 20
 end

data/lib/coderay.rb CHANGED Viewed

@@ -1,6 +1,6 @@
 # = CodeRay Library
 #
-# $Id: coderay.rb 44 2005-10-01 00:59:05Z murphy $
+# $Id: coderay.rb 73 2005-10-29 16:09:17Z murphy $
 #
 # CodeRay is a Ruby library for syntax highlighting.
 #
@@ -128,7 +128,7 @@
 #	The scanning methods provide more flexibility; we recommend to use these.
 module CodeRay
-	Version = '0.4.4'
+	Version = '0.4.6'
 	require 'coderay/tokens'
 	require 'coderay/scanner'
@@ -201,6 +201,16 @@ module CodeRay
 			encoder(format, options).encode code, lang, options
 		end
+		# Highlight a string into a HTML <div>.
+		#
+		# CSS styles use classes, so you have to include a stylesheet
+		# in your output.
+		#
+		# See encode.
+		def highlight code, lang, options = { :css => :class }, format = :div
+			encode code, lang, format, options
+		end
 		# Encode pre-scanned Tokens.
 		# Use this together with CodeRay.scan:
 		#
@@ -223,10 +233,20 @@ module CodeRay
 		#  require 'coderay'
 		#  page = CodeRay.encode_file 'some_c_code.c', :html
 		def encode_file filename, format, options = {}
-			tokens = scan_file filename, auto, get_scanner_options(options)
+			tokens = scan_file filename, :auto, get_scanner_options(options)
 			encode_tokens tokens, format, options
 		end
+		# Highlight a file into a HTML <div>.
+		#
+		# CSS styles use classes, so you have to include a stylesheet
+		# in your output.
+		#
+		# See encode.
+		def highlight_file filename, options = { :css => :class }, format = :div
+			encode_file filename, format, options
+		end
 		# Finds the Encoder class for +format+ and creates an instance, passing
 		# +options+ to it.
 		#

data/lib/coderay/encoder.rb CHANGED Viewed

@@ -109,6 +109,11 @@ module CodeRay
 				method(:token).to_proc
 			end
+			# Return the default file extension for outputs of this encoder.
+			def file_extension
+				self.class::FILE_EXTENSION
+			end
 		protected
 			# Called with merged options before encoding starts.
@@ -122,11 +127,28 @@ module CodeRay
 			# Called with +text+ and +kind+ of the currently scanned token.
 			# For simple scanners, it's enougth to implement this method.
 			#
-			# Raises a NotImplementedError exception if it is not overwritten
-			# in subclass.
+			# By default, it calls text_token or block_token, depending on
+			# whether +text+ is a String.
 			def token text, kind
-				raise NotImplementedError,
-					"#{self.class}#token not implemented."
+				if text.is_a? String
+					text_token text, kind
+				else
+					block_token text, kind
+				end
+			end
+			def text_token text, kind
+			end
+			def block_token action, kind
+				case action
+				when :open
+					open_token kind
+				when :close
+					close_token kind
+				else
+					raise 'unknown block action: %p' % action
+				end
 			end
 			# Called with merged options after encoding starts.

data/lib/coderay/encoders/debug.rb ADDED Viewed

@@ -0,0 +1,38 @@
+module CodeRay
+	module Encoders
+		# = Debug Encoder
+		class Debug < Encoder
+			include Streamable
+			register_for :debug
+			FILE_EXTENSION = 'raydebug'
+		protected
+			def text_token text, kind
+				@out <<
+					if kind == :space
+						text
+					else
+						text = text.gsub(/[)\\]/, '\\\\\0')
+						"#{kind}(#{text})"
+					end
+			end
+			def block_token action, kind
+				@out << super
+			end
+			def open_token kind
+				"#{kind}<"
+			end
+			def close_token kind
+				">"
+			end
+		end
+	end
+end

data/lib/coderay/encoders/helpers/html_css.rb CHANGED Viewed

@@ -102,6 +102,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
 .hx { color:#058; font-weight:bold; }
 .i  { color:#00D; font-weight:bold; }
 .ic { color:#B44; font-weight:bold; }
+.il { }
 .in { color:#B2B; font-weight:bold; }
 .iv { color:#33B; }
 .la { color:#970; font-weight:bold; }

data/lib/coderay/encoders/helpers/html_helper.rb CHANGED Viewed

@@ -28,6 +28,7 @@ module CodeRay module Encoders
 			:global_variable => 'gv',
 			:hex => 'hx',
 			:include => 'ic',
+			:inline => 'il',
 			:instance_variable => 'iv',
 			:integer => 'i',
 			:interpreted => 'in',

data/lib/coderay/encoders/helpers/html_output.rb CHANGED Viewed

@@ -13,6 +13,8 @@ module CodeRay
 		# TODO: more doc.
 		module Output
+			attr_accessor :wrapped_in
 			class << self
 				# This makes Output look like a class.
@@ -33,8 +35,18 @@ module CodeRay
 					warn "The Output module is intended to extend instances of String, not #{o.class}." unless o.respond_to? :to_str
 				end
+				def stylesheet in_tag = false
+					ss = CSS::DEFAULT_STYLESHEET
+					ss = <<-CSS if in_tag
+<style type="text/css">
+#{ss}
+</style>
+					CSS
+					ss
+				end
 				def page_template_for_css css = :default
-					css = CSS::DEFAULT_STYLESHEET if css == :default
+					css = stylesheet if css == :default
 					PAGE.apply 'CSS', css
 				end
@@ -53,11 +65,6 @@ module CodeRay
 			wrapper :div, :span, :page
-			def wrapped_in
-				@wrapped_in ||= nil
-			end
-			attr_writer :wrapped_in
 			def wrapped_in? element
 				wrapped_in == element
 			end
@@ -97,19 +104,28 @@ module CodeRay
 				clone.wrap!(*args)
 			end
+			NUMERIZABLE_WRAPPINGS = {
+				:table => [:div, :page],
+				:inline => :all,
+				nil => :all
+			}
 			def numerize! mode = :table, options = {}
 				return self unless mode
-				start = options.fetch :line_number_start, DEFAULT_OPTIONS[:line_number_start]
+				options = DEFAULT_OPTIONS.merge options
+				start = options[:line_number_start]
 				unless start.is_a? Integer
 					raise ArgumentError, "Invalid value %p for :line_number_start; Integer expected." % start
 				end
-				unless NUMERIZABLE_WRAPPINGS.include? options[:wrap]
+				allowed_wrappings = NUMERIZABLE_WRAPPINGS[mode]
+				unless allowed_wrappings == :all or allowed_wrappings.include? options[:wrap]
 					raise ArgumentError, "Can't numerize, :wrap must be in %p, but is %p" % [NUMERIZABLE_WRAPPINGS, options[:wrap]]
 				end
-				bold_every = options.fetch :bold_every, DEFAULT_OPTIONS[:bold_every]
+				bold_every = options[:bold_every]
 				bolding =
 					if bold_every == :no_bolding or bold_every == 0
 						proc { |line| line.to_s }
@@ -137,7 +153,7 @@ module CodeRay
 						line += 1
 						"<span class=\"no\">#{ line_number.rjust(max_width) }</span>  "
 					end
-					wrap! :div
+					#wrap! :div
 				when :table
 					# This is really ugly.
@@ -164,6 +180,10 @@ module CodeRay
 				clone.numerize!(*args)
 			end
+			def stylesheet in_tag = false
+				Output.stylesheet in_tag
+			end
 			class Template < String
 				def self.wrap! str, template, target
@@ -207,9 +227,10 @@ module CodeRay
 			DIV_TABLE = <<-`DIV_TABLE`
 <table class="CodeRay"> <tr>
 	<td class="line_numbers" title="click to toggle" onclick="with (this.firstChild.style) { display = (display == '') ? 'none' : '' }"><pre><%LINE_NUMBERS%></pre></td>
-	<td class="code"><pre title="double click to expand" ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
+	<td class="code"><pre ondblclick="with (this.style) { overflow = (overflow == 'auto' || overflow == '') ? 'visible' : 'auto' }"><%CONTENT%></pre></td>
 </tr> </table>
 			DIV_TABLE
+			# title="double click to expand"
 			PAGE = <<-`PAGE`
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"

data/lib/coderay/encoders/html.rb CHANGED Viewed

@@ -1,6 +1,63 @@
 module CodeRay
 module Encoders
+	# = HTML Encoder
+	#
+	# This is CodeRay's most important highlighter:
+	# It provides save, fast XHTML generation and CSS support.
+	#
+	# == Usage
+	#
+	#  require 'coderay'
+	#  puts CodeRay.scan('Some /code/', :ruby).html  #-> a HTML page
+	#  puts CodeRay.scan('Some /code/', :ruby).html(:wrap => :span) #-> <span class="CodeRay"><span class="co">Some</span> /code/</span>
+	#  puts CodeRay.scan('Some /code/', :ruby).span  #-> the same
+	#
+	#  puts CodeRay.scan('Some code', :ruby).html(
+	#    :wrap => nil,
+	#    :line_numbers => :inline,
+	#    :css => :style
+	#  )
+	#  #-> <span class="no">1</span>  <span style="color:#036; font-weight:bold;">Some</span> code
+	#
+	# == Options
+	#
+	# === :tab_width
+	# Convert \t characters to +n+ spaces (a number.)
+	# Default: 8
+	#
+	# === :css
+	# How to include the styles; can be :class or :style.
+	#
+	# Default: :class
+	#
+	# === :wrap
+	# Wrap in :page, :div, :span or nil.
+	#
+	# You can also use Encoders::Div and Encoders::Span.
+	#
+	# Default: :page
+	#
+	# === :line_numbers
+	# Include line numbers in :table, :inline or nil (no line numbers)
+	#
+	# Default: nil
+	#
+	# === :line_number_start
+	# Where to start with line number counting.
+	#
+	# Default: 1
+	#
+	# === :bold_every
+	# Make every +n+-th number appear bold.
+	#
+	# Default: 10
+	#
+	# === :hint
+	# Include some information into the output using the title attribute.
+	# Can be :info (show token type on mouse-over) or :debug.
+	#
+	# Default: false
 	class HTML < Encoder
 		include Streamable
@@ -19,8 +76,9 @@ module Encoders
 			:line_numbers => nil,
 			:line_number_start => 1,
 			:bold_every => 10,
+			:hint => false,
 		}
-		NUMERIZABLE_WRAPPINGS = [:div, :page]
 		require 'coderay/encoders/helpers/html_helper'
 		require 'coderay/encoders/helpers/html_output'
@@ -47,12 +105,9 @@ module Encoders
 		ansi_chars = Array(0x7f..0xff)
 		ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
 		# \x9 (\t) and \xA (\n) not included
-		HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/
+		HTML_ESCAPE_PATTERN = /[\t&"><\xB-\x1f\x7f-\xff\0-\x8]/
 		def setup options
-			if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap]
-				warn ':line_numbers wanted, but :wrap is %p' % options[:wrap]
-			end
 			super
 			return if options == @last_options
 			@last_options = options
@@ -63,6 +118,11 @@ module Encoders
 			@opened = [nil]
 			@css = CSS.new
+			hint = options[:hint]
+			if hint and not [:debug, :info].include? hint
+				raise ArgumentError, "Unknown value %p for :hint; expected :info, :debug, false or nil." % hint
+			end
 			case options[:css]
 			when :class
@@ -73,15 +133,20 @@ module Encoders
 						type = k
 					end
 					c = ClassOfKind[type]
-					if c == :NO_HIGHLIGHT
+					if c == :NO_HIGHLIGHT and not hint
 						h[k] = false
 					else
-						if options[:debug]
-							debug_info = ' title="%p"' % [ k ]
+						title = if hint
+							if hint == :debug
+								' title="%p"' % [ k ]
+							elsif hint == :info
+								path = (k[1..-1] << k.first).map { |kind| kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize } }
+								" title=\"#{path.join('/')}\""
+							end
 						else
-							debug_info = ''
+							''
 						end
-						h[k] = '<span%s class="%s">' % [debug_info, c]
+						h[k] = '<span%s class="%s">' % [title, c]
 					end
 				end
@@ -92,19 +157,24 @@ module Encoders
 					else
 						styles = [k]
 					end
+					type = styles.first
 					styles.map! { |c| ClassOfKind[c] }
-					if styles.first == :NO_HIGHLIGHT
+					if styles.first == :NO_HIGHLIGHT and not hint
 						h[k] = false
 					else
-						if options[:debug]
-							debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ]
+						title = if hint
+							if hint == :debug
+								' title="%p"' % [ styles ]
+							elsif hint == :info and styles.size == 1
+								" title=\"#{type.to_s.gsub(/_/, " ").capitalize}\""
+							end
 						else
-							debug_info = ''
+							''
 						end
 						style = @css[*styles]
 						h[k] =
 							if style
-								'<span%s style="%s">' % [debug_info, style]
+								'<span%s style="%s">' % [title, style]
 							else
 								false
 							end
@@ -112,7 +182,7 @@ module Encoders
 				end
 			else
-				raise "Unknown value %p for :css." % options[:css]
+				raise ArgumentError, "Unknown value %p for :css." % options[:css]
 			end
 		end

data/lib/coderay/encoders/null.rb CHANGED Viewed

@@ -1,12 +1,20 @@
 module CodeRay
 	module Encoders
+		# = Null Encoder
+		#
+		# Does nothing and returns an empty string.
 		class Null < Encoder
 			include Streamable
 			register_for :null
-			protected
+			# Defined for faster processing
+			def to_proc
+				proc {}
+			end
+		protected
 			def token(*)
 				# do nothing
@@ -16,5 +24,3 @@ module CodeRay
 	end
 end

data/lib/coderay/encoders/statistic.rb CHANGED Viewed

@@ -22,17 +22,22 @@ module CodeRay module Encoders
 			super
 		end
-		def token text, type
+		def text_token text, kind
+			@real_token_count += 1 unless kind == :space
+			@type_stats[kind].count += 1
+			@type_stats[kind].size += text.size
+			@type_stats['TOTAL'].size += text.size
+		end
+		# TODO Hierarchy handling
+		def block_token action, kind
+			#@content_type = kind
+			@type_stats['open/close'].count += 1
+		end
+		def token text, kind
+			super
 			@type_stats['TOTAL'].count += 1
-			if text.is_a? String
-				@real_token_count += 1 unless type == :space
-				@type_stats[type].count += 1
-				@type_stats[type].size += text.size
-				@type_stats['TOTAL'].size += text.size
-			else
-				@content_type = type
-				@type_stats['open/close'].count += 1
-			end
 		end
 		STATS = <<-STATS

data/lib/coderay/encoders/xml.rb ADDED Viewed

@@ -0,0 +1,70 @@
+module CodeRay
+	module Encoders
+		# = Debug Encoder
+		class XML < Encoder
+			include Streamable
+			register_for :xml
+			FILE_EXTENSION = 'xml'
+			require 'rexml/document'
+			DEFAULT_OPTIONS = {
+				:tab_width => 8,
+				:pretty => -1,
+				:transitive => false,
+			}
+		protected
+			def setup options
+				@out = ''
+				@doc = REXML::Document.new
+				@doc << REXML::XMLDecl.new
+				@tab_width = options[:tab_width]
+				@root = @node = @doc.add_element('coderay-tokens')
+			end
+			def finish options
+				@doc.write @out, options[:pretty], options[:transitive], true
+				@out
+			end
+			def text_token text, kind
+				if kind == :space
+					token = @node
+				else
+					token = @node.add_element kind.to_s
+				end
+				text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
+					case
+					when space
+						token << REXML::Text.new(space, true)
+					when tab
+						token << REXML::Text.new(tab, true)
+					when nl
+						token << REXML::Text.new(nl, true)
+					else
+						token << REXML::Text.new($&)
+					end
+				end
+			end
+			def open_token kind
+				@node = @node.add_element kind.to_s
+			end
+			def close_token kind
+				if @node == @root
+					raise 'no token to close!'
+				end
+				@node = @node.parent
+			end
+		end
+	end
+end

data/lib/coderay/helpers/plugin.rb CHANGED Viewed

@@ -1,6 +1,6 @@
 # = PluginHost
 #
-# $Id: plugin.rb 47 2005-10-01 06:04:52Z murphy $
+# $Id: plugin.rb 59 2005-10-29 04:55:15Z murphy $
 #
 # A simple subclass plugin system.
 #
@@ -90,10 +90,10 @@ module PluginHost
 				id = validate_id(plugin_id)
 				path = path_to id
 				begin
-					$stderr.puts 'Loading plugin: ' + path if $DEBUG
+					#$stderr.puts 'Loading plugin: ' + path if $DEBUG
 					require path
-				rescue LoadError
-					raise PluginNotFound, "Plugin #{id.inspect} not found."
+				rescue LoadError => boom
+					raise PluginNotFound, 'Could not load plugin %p: %s' % [id, boom]
 				else
 					# Plugin should have registered by now
 					unless h.has_key? id
@@ -151,7 +151,7 @@ module PluginHost
 	end
 	# Alias for +[]+.
-	alias load_plugin []
+	alias load []
 	# Returns the Plugin for +id+.
 	# Use it like Hash#fetch.
@@ -239,7 +239,7 @@ def require_plugin path
 	host = PluginHost.host_by_id(host_id)
 	raise PluginHost::HostNotFound,
 		"No host for #{host_id.inspect} found." unless host
-	host.load_plugin plugin_id
+	host.load plugin_id
 end

data/lib/coderay/scanners/helpers/ruby_helper.rb CHANGED Viewed

@@ -12,9 +12,11 @@ module CodeRay module Scanners
 		]
 		DEF_KEYWORDS = %w[ def ]
+		UNDEF_KEYWORDS = %w[ undef ]
 		MODULE_KEYWORDS = %w[class module]
 		DEF_NEW_STATE = WordList.new(:initial).
 			add(DEF_KEYWORDS, :def_expected).
+			add(UNDEF_KEYWORDS, :undef_expected).
 			add(MODULE_KEYWORDS, :module_expected)
 		IDENTS_ALLOWING_REGEXP = %w[
@@ -32,13 +34,11 @@ module CodeRay module Scanners
 			add(RESERVED_WORDS, :reserved).
 			add(PREDEFINED_CONSTANTS, :pre_constant)
-#		IDENT = /[a-zA-Z_][a-zA-Z_0-9]*/
 		IDENT = /[a-z_][\w_]*/i
 		METHOD_NAME = / #{IDENT} [?!]? /ox
-		METHOD_NAME_EX = /
-			#{IDENT}[?!=]?  # common methods: split, foo=, empty?, gsub!
-			| \*\*?         # multiplication and power
+		METHOD_NAME_OPERATOR = /
+			\*\*?           # multiplication and power
 			| [-+]@?        # plus, minus
 			| [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
 			| \[\]=?        # array getter and setter
@@ -46,10 +46,11 @@ module CodeRay module Scanners
 			| <=?>? | >=?   # comparison, rocket operator
 			| ===?          # simple equality and case equality
 		/ox
+		METHOD_NAME_EX = / #{IDENT} [?!=]? | #{METHOD_NAME_OPERATOR} /ox
 		INSTANCE_VARIABLE = / @ #{IDENT} /ox
 		CLASS_VARIABLE = / @@ #{IDENT} /ox
 		OBJECT_VARIABLE = / @@? #{IDENT} /ox
-		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9] | 0[a-zA-Z_0-9]* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
 		PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
 		VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
@@ -60,19 +61,17 @@ module CodeRay module Scanners
 		QUOTE_TO_TYPE.default = :string
 		REGEXP_MODIFIERS = /[mixounse]*/
-		REGEXP_SYMBOLS = /
-			[|?*+?(){}\[\].^$]
-		/x
+		REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/
-		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+		DECIMAL = /\d+(?:_\d+)*/
 		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
 		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
 		BINARY = /0b[01]+(?:_[01]+)*/
 		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
-		FLOAT_OR_INT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? )? /ox
-		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /ox
-		NUMERIC = / #{OCTAL} | #{HEXADECIMAL} | #{BINARY} | #{FLOAT_OR_INT} /ox
+		FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
+		FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
+		NUMERIC = / (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} /ox
 		SYMBOL = /
 			:
@@ -103,7 +102,7 @@ module CodeRay module Scanners
 			)
 		/mx
-		# NOTE: This is not completel correct, but
+		# NOTE: This is not completely correct, but
 		# nobody needs heredoc delimiters ending with \n.
 		HEREDOC_OPEN = /
 			<< (-)?              # $1 = float
@@ -115,7 +114,7 @@ module CodeRay module Scanners
 			)
 		/mx
-		RDOC = /
+		RUBYDOC = /
 			=begin (?!\S)
 			.*?
 			(?: \Z | ^=end (?!\S) [^\n]* )
@@ -127,6 +126,8 @@ module CodeRay module Scanners
 			(?: \Z | (?=^\#CODE) )
 		/mx
+		RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
 		RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
 		FANCY_START = / % ( [qQwWxsr] | (?![\w\s=]) ) (.) /mox
@@ -144,7 +145,7 @@ module CodeRay module Scanners
 		FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
 		class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
-			:paren, :paren_depth, :pattern
+			:paren, :paren_depth, :pattern, :next_state
 			CLOSING_PAREN = Hash[ *%w[
 				( )
@@ -186,7 +187,7 @@ module CodeRay module Scanners
 				delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
 				h[k] =
 					if interpreted
-						/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx
+				/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx  # $1 set == end of heredoc
 					else
 						/ (?= #{delim_pattern}() | \\ ) /mx
 					end
@@ -203,7 +204,7 @@ module CodeRay module Scanners
 				else
 					pattern = STRING_PATTERN[ [delim, interpreted] ]
 				end
-				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern
+				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern, :initial
 			end
 		end unless defined? StringState

data/lib/coderay/scanners/ruby.rb CHANGED Viewed

@@ -43,7 +43,6 @@ module CodeRay module Scanners
 				if state.instance_of? StringState
 # {{{
 					match = scan_until(state.pattern) || scan_until(/\z/)
 					tokens << [match, :content] unless match.empty?
 					break if eos?
@@ -52,7 +51,7 @@ module CodeRay module Scanners
 						match = getch + scan_until(/$/)
 						tokens << [match, :delimiter]
 						tokens << [:close, state.type]
-						state = :initial
+						state = state.next_state
 						next
 					end
@@ -103,7 +102,7 @@ module CodeRay module Scanners
 						end
 						tokens << [:close, state.type]
 						fancy_allowed = regexp_allowed = false
-						state = :initial
+						state = state.next_state
 					when '\\'
 						if state.interpreted
@@ -128,13 +127,14 @@ module CodeRay module Scanners
 							fancy_allowed = regexp_allowed = true
 							state = :initial
 							depth = 1
-							tokens << [match + getch, :escape]
+							tokens << [:open, :inline]
+							tokens << [match + getch, :delimiter]
 						when ?$, ?@
 							tokens << [match, :escape]
 							last_state = state  # scan one token as normal code, then return here
 							state = :initial
 						else
-							raise "else-case # reached; #%p not handled" % peek(1), tokens
+							raise 'else-case # reached; #%p not handled' % peek(1), tokens
 						end
 					when state.paren
@@ -145,7 +145,7 @@ module CodeRay module Scanners
 						tokens << [match, :function]
 					else
-						raise "else-case \" reached; %p not handled, state = %p" % [match, state], tokens
+						raise 'else-case " reached; %p not handled, state = %p' % [match, state], tokens
 					end
 					next
@@ -153,7 +153,7 @@ module CodeRay module Scanners
 				else
 # {{{
 					if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
-						( bol? and match = scan(/ #{DATA} | #{RDOC} /ox) )
+						( bol? and match = scan(/#{RUBYDOC_OR_DATA}/o) )
 						fancy_allowed = true
 						case m = match[0]
 						when ?\s, ?\t, ?\f
@@ -161,7 +161,10 @@ module CodeRay module Scanners
 							type = :space
 						when ?\n, ?\\
 							type = :space
-							regexp_allowed = m == ?\n
+							if m == ?\n
+								regexp_allowed = true
+								state = :initial if state == :undef_comma_expected
+							end
 							if heredocs
 								unscan  # heredoc scanning needs \n at start
 								state = heredocs.shift
@@ -175,14 +178,32 @@ module CodeRay module Scanners
 							type = :comment
 							regexp_allowed = true
 						else
-							raise "else-case _ reached, because case %p was not handled" % [matched[0].chr], tokens
+							raise 'else-case _ reached, because case %p was not handled' % [matched[0].chr], tokens
 						end
 						tokens << [match, type]
 						next
 					elsif state == :initial
-						if match = scan(/ \.\.?\.? | [-+*=>;,|&!\(\)\[\]~^]+ | [\{\}] | :: /x)
-							if match !~ / [.\)\]\}] \z/x or match =~ /\.\.\.?/
+						# IDENTS #
+						if match = scan(/#{METHOD_NAME}/o)
+							if last_token_dot
+								type = if match[/^[A-Z]/] then :constant else :ident end
+							else
+								type = IDENT_KIND[match]
+								if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]
+									type = :constant
+								elsif type == :reserved
+									state = DEF_NEW_STATE[match]
+								end
+							end
+							## experimental!
+							fancy_allowed = regexp_allowed = :set if REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
+						# OPERATORS #
+						elsif (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)) or
+							(last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))
+							if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
 								regexp_allowed = fancy_allowed = :set
 							end
 							last_token_dot = :set if match == '.' or match == '::'
@@ -195,33 +216,22 @@ module CodeRay module Scanners
 									depth -= 1
 									if depth == 0
 										state, depth, heredocs = states.pop
-										type = :escape
+										tokens << [match, :delimiter]
+										type = :inline
+										match = :close
 									end
 								end
 							end
-						elsif match = scan(/#{METHOD_NAME}/o)
-							if last_token_dot
-								type = if match[/^[A-Z]/] then :constant else :ident end
-							else
-								type = IDENT_KIND[match]
-								if type == :ident and match[/^[A-Z]/]
-									type = :constant
-								elsif type == :reserved
-									state = DEF_NEW_STATE[match]
-								end
-							end
-							fancy_allowed = regexp_allowed = REGEXP_ALLOWED[match]
 						elsif match = scan(/ ['"] /mx)
 							tokens << [:open, :string]
 							type = :delimiter
-							state = StringState.new :string, match != '\'', match  # important for streaming
+							state = StringState.new :string, match == '"', match  # important for streaming
 						elsif match = scan(/#{INSTANCE_VARIABLE}/o)
 							type = :instance_variable
-						elsif regexp_allowed and match = scan(/ \/ /mx)
+						elsif regexp_allowed and match = scan(/\//)
 							tokens << [:open, :regexp]
 							type = :delimiter
 							interpreted = true
@@ -232,15 +242,23 @@ module CodeRay module Scanners
 							end
 						elsif match = scan(/#{NUMERIC}/o)
-							type = if match[/#{FLOAT}/o] then :float else :integer end
+							type = if self[1] then :float else :integer end
-						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)
-							case match[1]
+						elsif match = scan(/#{SYMBOL}/o)
+							case delim = match[1]
 							when ?', ?"
 								tokens << [:open, :symbol]
-								state = StringState.new :symbol, match[1] == ?", match[1,1]
+								tokens << [':', :symbol]
+								match = delim.chr
+								type = :delimiter
+								state = StringState.new :symbol, delim == ?", match
+							else
+								type = :symbol
 							end
-							type = :symbol
+						elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
+							regexp_allowed = fancy_allowed = :set
+							type = :operator
 						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
 							indented = self[1] == '-'
@@ -265,7 +283,7 @@ module CodeRay module Scanners
 						elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
 							type = :integer
-						elsif match = scan(/ [\/%<?:] /x)
+						elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
 							regexp_allowed = fancy_allowed = :set
 							type = :operator
@@ -290,25 +308,55 @@ module CodeRay module Scanners
 						end
 					elsif state == :def_expected
-						if match = scan(/ (?: #{VARIABLE} (?: ::#{IDENT} )* \. )? #{METHOD_NAME_EX} /ox)
+						state = :initial
+						if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)
 							type = :method
 						else
-							match = getch
+							next
+						end
+					elsif state == :undef_expected
+						state = :undef_comma_expected
+						if match = scan(/#{METHOD_NAME_EX}/o)
+							type = :method
+						elsif match = scan(/#{SYMBOL}/o)
+							case delim = match[1]
+							when ?', ?"
+								tokens << [:open, :symbol]
+								tokens << [':', :symbol]
+								match = delim.chr
+								type = :delimiter
+								state = StringState.new :symbol, delim == ?", match
+								state.next_state = :undef_comma_expected
+							else
+								type = :symbol
+							end
+						else
+							state = :initial
+							next
+						end
+					elsif state == :undef_comma_expected
+						if match = scan(/,/)
+							type = :operator
+							state = :undef_expected
+						else
+							state = :initial
+							next
 						end
-						state = :initial
 					elsif state == :module_expected
 						if match = scan(/<</)
 							type = :operator
 						else
+							state = :initial
 							if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
 								type = :class
 							else
-								match = getch
+								next
 							end
 						end
-						state = :initial
 					end
 					regexp_allowed = regexp_allowed == :set
@@ -316,7 +364,7 @@ module CodeRay module Scanners
 					last_token_dot = last_token_dot == :set
 					if $DEBUG
-						raise_inspect 'error token %p in line %d' % [tokens.last, line], tokens if not type or type == :error
+						raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error
 					end
 					tokens << [match, type]

data/lib/coderay/tokens.rb CHANGED Viewed

@@ -94,15 +94,14 @@ module CodeRay
 			false
 		end
-		alias :orig_each :each
 		# Iterates over all tokens.
 		#
 		# If a filter is given, only tokens of that kind are yielded.
 		def each kind_filter = nil, &block
 			unless kind_filter
-				orig_each(&block)
+				super(&block)
 			else
-				orig_each do |text, kind|
+				super do |text, kind|
 					next unless kind == kind_filter
 					yield text, kind
 				end
@@ -115,7 +114,7 @@ module CodeRay
 		# Example:
 		#   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
 		def each_text_token
-			orig_each do |text, kind|
+			each do |text, kind|
 				next unless text.respond_to? :to_str
 				yield text, kind
 			end

metadata CHANGED Viewed

@@ -3,8 +3,8 @@ rubygems_version: 0.8.10
 specification_version: 1
 name: coderay
 version: !ruby/object:Gem::Version
-  version: 0.4.3.48
-date: 2005-10-01
+  version: 0.4.5.73
+date: 2005-10-29
 summary: CodeRay is a fast syntax highlighter engine for many languages.
 require_paths:
   - lib
@@ -31,47 +31,50 @@ authors:
   - murphy
 files:
   - "./lib/coderay.rb"
-  - "./lib/coderay/encoder.rb"
-  - "./lib/coderay/scanner.rb"
   - "./lib/coderay/tokens.rb"
-  - "./lib/coderay/encoders/yaml.rb"
+  - "./lib/coderay/scanner.rb"
+  - "./lib/coderay/encoder.rb"
+  - "./lib/coderay/encoders/xml.rb"
+  - "./lib/coderay/encoders/span.rb"
   - "./lib/coderay/encoders/div.rb"
+  - "./lib/coderay/encoders/yaml.rb"
   - "./lib/coderay/encoders/tokens.rb"
+  - "./lib/coderay/encoders/text.rb"
   - "./lib/coderay/encoders/statistic.rb"
+  - "./lib/coderay/encoders/count.rb"
   - "./lib/coderay/encoders/null.rb"
-  - "./lib/coderay/encoders/span.rb"
+  - "./lib/coderay/encoders/debug.rb"
   - "./lib/coderay/encoders/html.rb"
-  - "./lib/coderay/encoders/text.rb"
-  - "./lib/coderay/encoders/count.rb"
   - "./lib/coderay/encoders/helpers/html_helper.rb"
-  - "./lib/coderay/encoders/helpers/html_output.rb"
   - "./lib/coderay/encoders/helpers/html_css.rb"
-  - "./lib/coderay/helpers/plugin.rb"
+  - "./lib/coderay/encoders/helpers/html_output.rb"
   - "./lib/coderay/helpers/filetype.rb"
-  - "./lib/coderay/helpers/gzip_simple.rb"
+  - "./lib/coderay/helpers/plugin.rb"
   - "./lib/coderay/helpers/scanner_helper.rb"
+  - "./lib/coderay/helpers/gzip_simple.rb"
   - "./lib/coderay/scanners/c.rb"
   - "./lib/coderay/scanners/delphi.rb"
   - "./lib/coderay/scanners/plaintext.rb"
   - "./lib/coderay/scanners/ruby.rb"
   - "./lib/coderay/scanners/helpers/ruby_helper.rb"
   - "./demo/demo_global_vars2.rb"
+  - "./demo/demo_stream2.rb"
+  - "./demo/demo_load_encoder.rb"
   - "./demo/demo_encoder.rb"
   - "./demo/demo_div.rb"
+  - "./demo/demo_html2.rb"
+  - "./demo/demo_css.rb"
   - "./demo/demo_count.rb"
   - "./demo/demo_simple.rb"
-  - "./demo/demo_tokens.rb"
-  - "./demo/demo_css.rb"
-  - "./demo/demo_html.rb"
-  - "./demo/demo_dump.rb"
-  - "./demo/demo_stream2.rb"
-  - "./demo/demo_load_encoder.rb"
-  - "./demo/demo_html2.rb"
   - "./demo/demo_scanner.rb"
   - "./demo/demo_stream.rb"
   - "./demo/demo_global_vars.rb"
   - "./demo/demo_more.rb"
+  - "./demo/demo_tokens.rb"
+  - "./demo/demo_html.rb"
+  - "./demo/demo_dump.rb"
   - "./demo/demo_server.rb"
+  - "./demo/demo_highlight.rb"
   - "./README"
   - "./LICENSE"
 test_files: []