coderay 0.5.0.121 → 0.7.1.147

data/lib/coderay/scanners/delphi.rb

@@ -1,4 +1,5 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
 	
 	class Delphi < Scanner
 
@@ -109,7 +110,11 @@ module CodeRay module Scanners
 				end
 				
 				match ||= matched
-				raise [match, kind], tokens if kind == :error
+				if $DEBUG and (not kind or kind == :error)
+					raise_inspect 'Error token %p in line %d' %
+						[[match, kind], line], tokens
+				end
+				raise_inspect 'Empty token', tokens unless match
 
 				tokens << [match, kind]
 				
@@ -120,4 +125,5 @@ module CodeRay module Scanners
 
 	end
 
-end end
+end
+end

data/lib/coderay/scanners/html.rb

@@ -0,0 +1,167 @@
+module CodeRay
+module Scanners
+
+	# HTML Scanner
+	#
+	# $Id$
+	class HTML < Scanner
+
+		include Streamable
+		register_for :html
+
+		ATTR_NAME = /[\w.:-]+/
+		ATTR_VALUE_UNQUOTED = ATTR_NAME
+		TAG_END = /\/?>/
+		HEX = /[0-9a-fA-F]/
+		ENTITY = /
+			&
+			(?:
+				\w+
+			|
+				\#
+				(?:
+					\d+
+				|
+					x#{HEX}+
+				)
+			)
+			;
+		/ox
+
+		PLAIN_STRING_CONTENT = {
+			"'" => /[^&'>\n]+/,
+			'"' => /[^&">\n]+/,
+		}
+
+	private
+		def setup
+			@state = :initial
+			@plain_string_content = nil
+		end
+		
+		def scan_tokens tokens, options
+
+			state = @state
+			plain_string_content = @plain_string_content
+			
+			until eos?
+				
+				kind = :error
+				match = nil
+
+				if scan(/\s+/m)
+					kind = :space
+					
+				else
+					
+					case state
+						
+					when :initial
+						if scan(/<!--.*?-->/m)
+							kind = :comment
+						elsif scan(/<!DOCTYPE.*?>/m)
+							kind = :preprocessor
+						elsif scan(/<\?xml.*?\?>/m)
+							kind = :preprocessor
+						elsif scan(/<\?.*?\?>|<%.*?%>/m)
+							kind = :comment
+						elsif scan(/<\/[-\w_.:]*>/m)
+							kind = :tag
+						elsif match = scan(/<[-\w_.:]*>?/m)
+							kind = :tag
+							state = :attribute unless match[-1] == ?>
+						elsif scan(/[^<>&]+/)
+							kind = :plain
+						elsif scan(/#{ENTITY}/ox)
+							kind = :entity
+						elsif scan(/[>&]/)
+							kind = :error
+						else
+							raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
+						end
+						
+					when :attribute
+						if scan(/#{TAG_END}/)
+							kind = :tag
+							state = :initial
+						elsif scan(/#{ATTR_NAME}/o)
+							kind = :attribute_name
+							state = :attribute_equal
+						else
+							getch
+						end
+
+					when :attribute_equal
+						if scan(/=/)
+							kind = :operator
+							state = :attribute_value
+						elsif scan(/#{ATTR_NAME}/o)
+							kind = :attribute_name
+						elsif scan(/#{TAG_END}/o)
+							kind = :tag
+							state = :initial
+						elsif scan(/./)
+							state = :attribute
+						end
+						
+					when :attribute_value
+						if scan(/#{ATTR_VALUE_UNQUOTED}/o)
+							kind = :attribute_value
+							state = :attribute
+						elsif match = scan(/["']/)
+							tokens << [:open, :string]
+							state = :attribute_value_string
+							plain_string_content = PLAIN_STRING_CONTENT[match]
+							kind = :delimiter
+						elsif scan(/#{TAG_END}/o)
+							kind = :tag
+							state = :initial
+						else
+							getch
+						end
+
+					when :attribute_value_string
+						if scan(plain_string_content)
+							kind = :content
+						elsif scan(/['"]/)
+							tokens << [matched, :delimiter]
+							tokens << [:close, :string]
+							state = :attribute
+							next
+						elsif scan(/#{ENTITY}/ox)
+							kind = :entity
+						elsif scan(/[\n>]/)
+							tokens << [:close, :string]
+							kind = :error
+							state = :initial
+						end
+
+					else
+						raise_inspect 'Unknown state: %p' % [state], tokens
+
+					end
+
+				end
+
+				match ||= matched
+				if $DEBUG and (not kind or kind == :error)
+					raise_inspect 'Error token %p in line %d' %
+						[[match, kind], line], tokens
+				end
+				raise_inspect 'Empty token', tokens unless match
+				
+				tokens << [match, kind]
+			end
+
+			if options[:keep_state]
+				@state = state
+				@plain_string_content = plain_string_content
+			end
+
+			tokens
+		end
+
+	end
+
+end
+end

data/lib/coderay/scanners/nitro_html.rb

@@ -0,0 +1,125 @@
+module CodeRay
+module Scanners
+
+	load :html
+	load :ruby
+
+	# RHTML Scanner
+	# 
+	# $Id$
+	class NitroHTML < Scanner
+
+		include Streamable
+		register_for :nitro_html
+
+		NITRO_RUBY_BLOCK = /
+			<\?r
+			(?>
+				[^\?]*
+				(?> \?(?!>) [^\?]* )* 
+			)
+			(?: \?> )?
+		|
+			<ruby>
+			(?>
+				[^<]*
+				(?> <(?!\/ruby>) [^<]* )* 
+			)
+			(?: <\/ruby> )?
+		|
+			<%
+			(?>
+				[^%]*
+				(?> %(?!>) [^%]* )*
+			)
+			(?: %> )?
+		/mx
+
+		NITRO_VALUE_BLOCK = /
+			\#
+			(?:
+				\{
+				[^{}]*
+				(?>
+					\{ [^}]* \}
+					(?> [^{}]* )
+				)*
+				\}?
+			| \| [^|]* \|?
+			| \( [^)]* \)?
+			| \[ [^\]]* \]?
+			| \\ [^\\]* \\?
+			)
+		/x
+
+		NITRO_ENTITY = /
+			% (?: \#\d+ | \w+ ) ;
+		/
+
+		START_OF_RUBY = /
+			(?=[<\#%])
+			< (?: \?r | % | ruby> )
+		| \# [{(|]
+		| % (?: \#\d+ | \w+ ) ;
+		/x		
+
+		CLOSING_PAREN = Hash.new do |h, p|
+			h[p] = p
+		end.update( {
+			'(' => ')',
+			'[' => ']',
+			'{' => '}',
+		} )
+		
+	private
+		
+		def setup
+			@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+			@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+		end
+		
+		def scan_tokens tokens, options
+			
+			until eos?
+
+				if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
+					@html_scanner.tokenize match
+					
+				elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
+					start_tag = match[0,2]
+					delimiter = CLOSING_PAREN[start_tag[1,1]]
+					end_tag = match[-1,1] == delimiter ? delimiter : ''
+					tokens << [:open, :inline]
+					tokens << [start_tag, :delimiter]
+					code = match[start_tag.size .. -1 - end_tag.size]
+					@ruby_scanner.tokenize code
+					tokens << [end_tag, :delimiter] unless end_tag.empty?
+					tokens << [:close, :inline]
+
+				elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
+					start_tag = '<?r'
+					end_tag = match[-2,2] == '?>' ? '?>' : ''
+					tokens << [:open, :inline]
+					tokens << [start_tag, :delimiter]
+					code = match[start_tag.size .. -(end_tag.size)-1]
+					@ruby_scanner.tokenize code
+					tokens << [end_tag, :delimiter] unless end_tag.empty?
+					tokens << [:close, :inline]
+
+				elsif entity = scan(/#{NITRO_ENTITY}/o)
+					tokens << [entity, :entity]
+
+				else
+					raise_inspect 'else-case reached!', tokens
+				end
+
+			end
+
+			tokens
+
+		end
+
+	end
+
+end
+end

data/lib/coderay/scanners/plaintext.rb

@@ -1,4 +1,5 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
 
 	class Plaintext < Scanner
 		
@@ -10,4 +11,5 @@ module CodeRay module Scanners
 
 	end
 
-end end
+end
+end

data/lib/coderay/scanners/rhtml.rb

@@ -0,0 +1,65 @@
+module CodeRay
+module Scanners
+
+	load :html
+	load :ruby
+
+	# RHTML Scanner
+	# 
+	# $Id$
+	class RHTML < Scanner
+
+		include Streamable
+		register_for :rhtml
+
+		ERB_RUBY_BLOCK = /
+			<%(?!%)[=-]?
+			(?>
+				[^%]*
+				(?> %(?!>) [^%]* )* 
+			)
+			(?: %> )?
+		/x
+
+		START_OF_ERB = /
+			<%(?!%)
+		/x		
+
+	private
+		
+		def setup
+			@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
+			@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+		end
+		
+		def scan_tokens tokens, options
+			
+			until eos?
+
+				if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
+					@html_scanner.tokenize match
+					
+				elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
+					start_tag = match[/\A<%[-=]?/]
+					end_tag = match[/%?>?\z/]
+					tokens << [:open, :inline]
+					tokens << [start_tag, :delimiter]
+					code = match[start_tag.size .. -1 - end_tag.size]
+					@ruby_scanner.tokenize code
+					tokens << [end_tag, :delimiter] unless end_tag.empty?
+					tokens << [:close, :inline]
+
+				else
+					raise_inspect 'else-case reached!', tokens
+				end
+
+			end
+
+			tokens
+
+		end
+
+	end
+
+end
+end

data/lib/coderay/scanners/ruby.rb

@@ -1,4 +1,5 @@
-module CodeRay module Scanners
+module CodeRay
+module Scanners
 
 	# This scanner is really complex, since Ruby _is_ a complex language!
 	#
@@ -36,12 +37,14 @@ module CodeRay module Scanners
 			depth = nil
 			states = []
 
+			patterns = Patterns  # avoid constant lookup
+
 			until eos?
 				type = :error
 				match = nil
 				kind = nil
 
-				if state.instance_of? StringState
+				if state.instance_of? patterns::StringState
 # {{{
 					match = scan_until(state.pattern) || scan_until(/\z/)
 					tokens << [match, :content] unless match.empty?
@@ -67,14 +70,14 @@ module CodeRay module Scanners
 						end
 						tokens << [match, :delimiter]
 						if state.type == :regexp and not eos?
-							modifiers = scan(/#{REGEXP_MODIFIERS}/ox)
+							modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
 							tokens << [modifiers, :modifier] unless modifiers.empty?
 							if parse_regexp
 								extended = modifiers.index ?x
 								tokens = saved_tokens
 								regexp = tokens
 								for text, type in regexp
-									if text.is_a? String
+									if text.is_a? ::String
 										case type
 										when :content
 											text.scan(/([^#]+)|(#.*)/) do |plain, comment|
@@ -106,7 +109,7 @@ module CodeRay module Scanners
 						
 					when '\\'
 						if state.interpreted
-							if esc = scan(/ #{ESCAPE} /ox)
+							if esc = scan(/ #{patterns::ESCAPE} /ox)
 								tokens << [match + esc, :char]
 							else
 								tokens << [match, :error]
@@ -141,7 +144,7 @@ module CodeRay module Scanners
 						state.paren_depth += 1
 						tokens << [match, :nesting_delimiter]
 
-					when REGEXP_SYMBOLS
+					when /#{patterns::REGEXP_SYMBOLS}/ox
 						tokens << [match, :function]
 						
 					else
@@ -153,7 +156,7 @@ module CodeRay module Scanners
 				else
 # {{{					
 					if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
-						( bol? and match = scan(/#{RUBYDOC_OR_DATA}/o) )
+						( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
 						fancy_allowed = true
 						case m = match[0]
 						when ?\s, ?\t, ?\f
@@ -186,23 +189,23 @@ module CodeRay module Scanners
 					elsif state == :initial
 						
 						# IDENTS #
-						if match = scan(/#{METHOD_NAME}/o)
+						if match = scan(/#{patterns::METHOD_NAME}/o)
 							if last_token_dot
-								type = if match[/^[A-Z]/] then :constant else :ident end
+								type = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
 							else
-								type = IDENT_KIND[match]
-								if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]
+								type = patterns::IDENT_KIND[match]
+								if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
 									type = :constant
 								elsif type == :reserved
-									state = DEF_NEW_STATE[match]
+									state = patterns::DEF_NEW_STATE[match]
 								end
 							end
 							## experimental!
-							fancy_allowed = regexp_allowed = :set if REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
+							fancy_allowed = regexp_allowed = :set if patterns::REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
 							
 						# OPERATORS #
 						elsif (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)) or
-							(last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))
+							(last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o))
 							if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
 								regexp_allowed = fancy_allowed = :set
 							end
@@ -226,32 +229,32 @@ module CodeRay module Scanners
 						elsif match = scan(/ ['"] /mx)
 							tokens << [:open, :string]
 							type = :delimiter
-							state = StringState.new :string, match == '"', match  # important for streaming
+							state = patterns::StringState.new :string, match == '"', match  # important for streaming
 							
-						elsif match = scan(/#{INSTANCE_VARIABLE}/o)
+						elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
 							type = :instance_variable
 
 						elsif regexp_allowed and match = scan(/\//)
 							tokens << [:open, :regexp]
 							type = :delimiter
 							interpreted = true
-							state = StringState.new :regexp, interpreted, match
+							state = patterns::StringState.new :regexp, interpreted, match
 							if parse_regexp
 								tokens = []
 								saved_tokens = tokens
 							end
 							
-						elsif match = scan(/#{NUMERIC}/o)
+						elsif match = scan(/#{patterns::NUMERIC}/o)
 							type = if self[1] then :float else :integer end
 
-						elsif match = scan(/#{SYMBOL}/o)
+						elsif match = scan(/#{patterns::SYMBOL}/o)
 							case delim = match[1]
 							when ?', ?"
 								tokens << [:open, :symbol]
 								tokens << [':', :symbol]
 								match = delim.chr
 								type = :delimiter
-								state = StringState.new :symbol, delim == ?", match
+								state = patterns::StringState.new :symbol, delim == ?", match
 							else
 								type = :symbol
 							end
@@ -260,27 +263,27 @@ module CodeRay module Scanners
 							regexp_allowed = fancy_allowed = :set
 							type = :operator
 							
-						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
+						elsif fancy_allowed and match = scan(/#{patterns::HEREDOC_OPEN}/o)
 							indented = self[1] == '-'
 							quote = self[3]
 							delim = self[quote ? 4 : 2]
-							type = QUOTE_TO_TYPE[quote]
+							type = patterns::QUOTE_TO_TYPE[quote]
 							tokens << [:open, type]
 							tokens << [match, :delimiter]
 							match = :close
-							heredoc = StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
+							heredoc = patterns::StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
 							heredocs ||= []  # create heredocs if empty
 							heredocs << heredoc
 							
-						elsif fancy_allowed and match = scan(/#{FANCY_START}/o)
-							type, interpreted = *FancyStringType.fetch(self[1]) do
+						elsif fancy_allowed and match = scan(/#{patterns::FANCY_START_SAVE}/o)
+							type, interpreted = *patterns::FancyStringType.fetch(self[1]) do
 								raise_inspect 'Unknown fancy string: %%%p' % k, tokens
 							end
 							tokens << [:open, type]
-							state = StringState.new type, interpreted, self[2]
+							state = patterns::StringState.new type, interpreted, self[2]
 							type = :delimiter
 
-						elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
+						elsif fancy_allowed and match = scan(/#{patterns::CHARACTER}/o)
 							type = :integer
 
 						elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
@@ -293,13 +296,13 @@ module CodeRay module Scanners
 							else
 								tokens << [:open, :shell]
 								type = :delimiter
-								state = StringState.new :shell, true, match
+								state = patterns::StringState.new :shell, true, match
 							end
 							
-						elsif match = scan(/#{GLOBAL_VARIABLE}/o)
+						elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
 							type = :global_variable
 							
-						elsif match = scan(/#{CLASS_VARIABLE}/o)
+						elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
 							type = :class_variable
 							
 						else
@@ -309,7 +312,7 @@ module CodeRay module Scanners
 						
 					elsif state == :def_expected
 						state = :initial
-						if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)
+						if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
 							type = :method
 						else
 							next
@@ -317,16 +320,16 @@ module CodeRay module Scanners
 
 					elsif state == :undef_expected
 						state = :undef_comma_expected
-						if match = scan(/#{METHOD_NAME_EX}/o)
+						if match = scan(/#{patterns::METHOD_NAME_EX}/o)
 							type = :method
-						elsif match = scan(/#{SYMBOL}/o)
+						elsif match = scan(/#{patterns::SYMBOL}/o)
 							case delim = match[1]
 							when ?', ?"
 								tokens << [:open, :symbol]
 								tokens << [':', :symbol]
 								match = delim.chr
 								type = :delimiter
-								state = StringState.new :symbol, delim == ?", match
+								state = patterns::StringState.new :symbol, delim == ?", match
 								state.next_state = :undef_comma_expected
 							else
 								type = :symbol
@@ -350,7 +353,7 @@ module CodeRay module Scanners
 							type = :operator
 						else
 							state = :initial
-							if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
+							if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
 								type = :class
 							else
 								next
@@ -358,14 +361,17 @@ module CodeRay module Scanners
 						end
 
 					end
+# }}}
 
 					regexp_allowed = regexp_allowed == :set
 					fancy_allowed = fancy_allowed == :set
 					last_token_dot = last_token_dot == :set
 
-					if $DEBUG
-						raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error
+					if $DEBUG and (not kind or kind == :error)
+						raise_inspect 'Error token %p in line %d' %
+							[[match, kind], line], tokens
 					end
+					raise_inspect 'Empty token', tokens unless match
 
 					tokens << [match, type]
 					
@@ -373,13 +379,19 @@ module CodeRay module Scanners
 						state = last_state
 						last_state = nil
 					end
-# }}}
 				end
 			end
 
+			states << state if state.is_a? patterns::StringState
+			until states.empty?
+				tokens << [:close, states.pop.type]
+			end
+
 			tokens
 		end
 	end
 
-end end
+end
+end
+
 # vim:fdm=marker