coderay 0.4.3.48

data/lib/coderay/scanners/helpers/ruby_helper.rb

@@ -0,0 +1,212 @@
+module CodeRay module Scanners
+
+	class Ruby
+
+		RESERVED_WORDS = %w[
+			and def end in or unless begin
+			defined? ensure module redo super until
+			BEGIN break do next rescue then
+			when END case else for retry
+			while alias class elsif if not return
+			undef yield
+		]
+
+		DEF_KEYWORDS = %w[ def ]
+		MODULE_KEYWORDS = %w[class module]
+		DEF_NEW_STATE = WordList.new(:initial).
+			add(DEF_KEYWORDS, :def_expected).
+			add(MODULE_KEYWORDS, :module_expected)
+
+		IDENTS_ALLOWING_REGEXP = %w[
+			and or not while until unless if then elsif when sub sub! gsub gsub! scan slice slice! split
+		]
+		REGEXP_ALLOWED = WordList.new(false).
+			add(IDENTS_ALLOWING_REGEXP, :set)
+		
+		PREDEFINED_CONSTANTS = %w[
+			nil true false self
+			DATA ARGV ARGF __FILE__ __LINE__
+		]
+
+		IDENT_KIND = WordList.new(:ident).
+			add(RESERVED_WORDS, :reserved).
+			add(PREDEFINED_CONSTANTS, :pre_constant)
+
+#		IDENT = /[a-zA-Z_][a-zA-Z_0-9]*/
+		IDENT = /[a-z_][\w_]*/i
+
+		METHOD_NAME = / #{IDENT} [?!]? /ox
+		METHOD_NAME_EX = /
+			#{IDENT}[?!=]?  # common methods: split, foo=, empty?, gsub!
+			| \*\*?         # multiplication and power
+			| [-+]@?        # plus, minus
+			| [\/%&|^`~]    # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
+			| \[\]=?        # array getter and setter
+			| << | >>       # append or shift left, shift right
+			| <=?>? | >=?   # comparison, rocket operator
+			| ===?          # simple equality and case equality
+		/ox
+		INSTANCE_VARIABLE = / @ #{IDENT} /ox
+		CLASS_VARIABLE = / @@ #{IDENT} /ox
+		OBJECT_VARIABLE = / @@? #{IDENT} /ox
+		GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9] | 0[a-zA-Z_0-9]* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
+		PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
+		VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
+
+		QUOTE_TO_TYPE = {
+			'`' => :shell,
+			'/'=> :regexp,
+		}
+		QUOTE_TO_TYPE.default = :string
+		
+		REGEXP_MODIFIERS = /[mixounse]*/
+		REGEXP_SYMBOLS = /
+			[|?*+?(){}\[\].^$]
+		/x
+
+		DECIMAL = /\d+(?:_\d+)*/  # doesn't recognize 09 as octal error
+		OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
+		HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
+		BINARY = /0b[01]+(?:_[01]+)*/
+
+		EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
+		FLOAT_OR_INT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? )? /ox
+		FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /ox
+		NUMERIC = / #{OCTAL} | #{HEXADECIMAL} | #{BINARY} | #{FLOAT_OR_INT} /ox
+
+		SYMBOL = /
+			:
+			(?:
+				#{METHOD_NAME_EX}
+			| #{PREFIX_VARIABLE}
+			| ['"]
+			)
+		/ox
+
+		# TODO investigste \M, \c and \C escape sequences
+		# (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
+		# assert_equal(225, ?\M-a)
+		# assert_equal(129, ?\M-\C-a)
+		ESCAPE = /
+				[abefnrstv]
+			| M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-
+			|	[0-7]{1,3}
+			| x[0-9A-Fa-f]{1,2}
+			| .
+		/mx
+
+		CHARACTER = /
+			\?
+			(?:
+				[^\s\\]
+			| \\ #{ESCAPE}
+			)
+		/mx
+
+		# NOTE: This is not completel correct, but
+		# nobody needs heredoc delimiters ending with \n.
+		HEREDOC_OPEN = /
+			<< (-)?              # $1 = float
+			(?:
+				( [A-Za-z_0-9]+ )  # $2 = delim
+			|
+				( ["'`] )          # $3 = quote, type
+				( [^\n]*? ) \3     # $4 = delim
+			)
+		/mx
+
+		RDOC = /
+			=begin (?!\S)
+			.*?
+			(?: \Z | ^=end (?!\S) [^\n]* )
+		/mx
+
+		DATA = /
+			__END__$
+			.*?
+			(?: \Z | (?=^\#CODE) )
+		/mx
+
+		RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
+
+		FANCY_START = / % ( [qQwWxsr] | (?![\w\s=]) ) (.) /mox
+
+		FancyStringType = {
+			'q' => [:string, false],
+			'Q' => [:string, true],
+			'r' => [:regexp, true],
+			's' => [:symbol, false],
+			'x' => [:shell, true],
+			'w' => [:string, :word],
+			'W' => [:string, :word],
+		}
+		FancyStringType['w'] = FancyStringType['q']
+		FancyStringType['W'] = FancyStringType[''] = FancyStringType['Q']
+			
+		class StringState < Struct.new :type, :interpreted, :delim, :heredoc,
+			:paren, :paren_depth, :pattern
+			
+			CLOSING_PAREN = Hash[ *%w[
+				( )
+				[ ]
+				< >
+				{ }
+			] ]
+			
+			CLOSING_PAREN.values.each { |o| o.freeze }  # debug, if I try to change it with <<
+			OPENING_PAREN = CLOSING_PAREN.invert
+
+			STRING_PATTERN = Hash.new { |h, k|
+				delim, interpreted = *k
+				delim_pattern = Regexp.escape(delim.dup)
+				if starter = OPENING_PAREN[delim]
+					delim_pattern << Regexp.escape(starter)
+				end
+
+				
+				special_escapes = 
+					case interpreted
+					when :regexp_symbols
+						'| ' + REGEXP_SYMBOLS.source
+					when :words
+						'| \s'
+					end
+
+				h[k] =
+					if interpreted and not delim == '#'
+						/ (?= [#{delim_pattern}\\] | \# [{$@] #{special_escapes} ) /mx
+					else
+						/ (?= [#{delim_pattern}\\] #{special_escapes} ) /mx
+					end
+			}
+
+			HEREDOC_PATTERN = Hash.new { |h, k|
+				delim, interpreted, indented = *k
+				delim_pattern = Regexp.escape(delim.dup)
+				delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
+				h[k] =
+					if interpreted
+						/ (?= #{delim_pattern}() | \\ | \# [{$@] ) /mx
+					else
+						/ (?= #{delim_pattern}() | \\ ) /mx
+					end
+			}
+
+			def initialize kind, interpreted, delim, heredoc = false
+				if paren = CLOSING_PAREN[delim]
+					delim, paren = paren, delim
+					paren_depth = 1
+				end
+				if heredoc
+					pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
+					delim	= nil
+				else
+					pattern = STRING_PATTERN[ [delim, interpreted] ]
+				end
+				super kind, interpreted, delim, heredoc, paren, paren_depth, pattern
+			end
+		end unless defined? StringState
+	
+	end
+
+end end

data/lib/coderay/scanners/plaintext.rb

@@ -0,0 +1,13 @@
+module CodeRay module Scanners
+
+	class Plaintext < Scanner
+		
+		register_for :plaintext, :plain
+
+		def scan_tokens tokens, options
+			tokens << [scan_until(/\z/), :plain]
+		end
+
+	end
+
+end end

data/lib/coderay/scanners/ruby.rb

@@ -0,0 +1,337 @@
+module CodeRay module Scanners
+
+	# This scanner is really complex, since Ruby _is_ a complex language!
+	#
+	# It tries to highlight 100% of all common code,
+	# and 90% of strange codes.
+	#
+	# It is optimized for HTML highlighting, and is not very useful for
+	# parsing or pretty printing.
+	#
+	# For now, I think it's better than the scanners in VIM or Syntax, or
+	# any highlighter I was able to find, except Caleb's RubyLexer.
+	#
+	# I hope it's also better than the rdoc/irb lexer.
+	class Ruby < Scanner
+
+		include Streamable
+
+		register_for :ruby
+
+		require 'coderay/scanners/helpers/ruby_helper'
+		
+		DEFAULT_OPTIONS = {
+			:parse_regexps => true,
+		}
+
+	private
+		def scan_tokens tokens, options
+			parse_regexp = false # options[:parse_regexps]
+			first_bake = saved_tokens = nil
+			last_token_dot = false
+			fancy_allowed = regexp_allowed = true
+			heredocs = nil
+			last_state = nil
+			state = :initial
+			depth = nil
+			states = []
+
+			until eos?
+				type = :error
+				match = nil
+				kind = nil
+
+				if state.instance_of? StringState
+# {{{
+
+					match = scan_until(state.pattern) || scan_until(/\z/)
+					tokens << [match, :content] unless match.empty?
+					break if eos?
+					
+					if state.heredoc and self[1]
+						match = getch + scan_until(/$/)
+						tokens << [match, :delimiter]
+						tokens << [:close, state.type]
+						state = :initial
+						next
+					end
+					
+					case match = getch
+					
+					when state.delim
+						if state.paren
+							state.paren_depth -= 1 
+							if state.paren_depth > 0
+								tokens << [match, :nesting_delimiter]
+								next
+							end
+						end
+						tokens << [match, :delimiter]
+						if state.type == :regexp and not eos?
+							modifiers = scan(/#{REGEXP_MODIFIERS}/ox)
+							tokens << [modifiers, :modifier] unless modifiers.empty?
+							if parse_regexp
+								extended = modifiers.index ?x
+								tokens = saved_tokens
+								regexp = tokens
+								for text, type in regexp
+									if text.is_a? String
+										case type
+										when :content
+											text.scan(/([^#]+)|(#.*)/) do |plain, comment|
+												if plain
+													tokens << [plain, :content]
+												else
+													tokens << [comment, :comment]
+												end
+											end
+										when :character
+											if text[/\\(?:[swdSWDAzZbB]|\d+)/]
+												tokens << [text, :modifier]
+											else
+												tokens << [text, type]
+											end
+										else
+											tokens << [text, type]
+										end
+									else
+										tokens << [text, type]
+									end										
+								end
+								first_bake = saved_tokens = nil
+							end
+						end
+						tokens << [:close, state.type]
+						fancy_allowed = regexp_allowed = false
+						state = :initial
+						
+					when '\\'
+						if state.interpreted
+							if esc = scan(/ #{ESCAPE} /ox)
+								tokens << [match + esc, :char]
+							else
+								tokens << [match, :error]
+							end
+						else
+							case m = getch
+							when state.delim, '\\'
+								tokens << [match + m, :char]
+							else
+								tokens << [match + m, :content]
+							end
+						end
+						
+					when '#'
+						case peek(1)[0]
+						when ?{
+							states.push [state, depth, heredocs]
+							fancy_allowed = regexp_allowed = true
+							state = :initial
+							depth = 1
+							tokens << [match + getch, :escape]
+						when ?$, ?@
+							tokens << [match, :escape]
+							last_state = state  # scan one token as normal code, then return here
+							state = :initial
+						else
+							raise "else-case # reached; #%p not handled" % peek(1), tokens
+						end
+						
+					when state.paren
+						state.paren_depth += 1
+						tokens << [match, :nesting_delimiter]
+
+					when REGEXP_SYMBOLS
+						tokens << [match, :function]
+						
+					else
+						raise "else-case \" reached; %p not handled, state = %p" % [match, state], tokens
+						
+					end
+					next
+# }}}
+				else
+# {{{					
+					if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
+						( bol? and match = scan(/ #{DATA} | #{RDOC} /ox) )
+						fancy_allowed = true
+						case m = match[0]
+						when ?\s, ?\t, ?\f
+							match << scan(/\s*/) unless eos? or heredocs
+							type = :space
+						when ?\n, ?\\
+							type = :space
+							regexp_allowed = m == ?\n
+							if heredocs
+								unscan  # heredoc scanning needs \n at start
+								state = heredocs.shift
+								tokens << [:open, state.type]
+								heredocs = nil if heredocs.empty?
+								next
+							else
+								match << scan(/\s*/) unless eos?
+							end
+						when ?#, ?=, ?_
+							type = :comment
+							regexp_allowed = true
+						else
+							raise "else-case _ reached, because case %p was not handled" % [matched[0].chr], tokens
+						end
+						tokens << [match, type]
+						next
+
+					elsif state == :initial
+						if match = scan(/ \.\.?\.? | [-+*=>;,|&!\(\)\[\]~^]+ | [\{\}] | :: /x)
+							if match !~ / [.\)\]\}] \z/x or match =~ /\.\.\.?/
+								regexp_allowed = fancy_allowed = :set
+							end
+							last_token_dot = :set if match == '.' or match == '::'
+							type = :operator
+							unless states.empty?
+								case match
+								when '{'
+									depth += 1
+								when '}'
+									depth -= 1
+									if depth == 0
+										state, depth, heredocs = states.pop
+										type = :escape
+									end
+								end
+							end
+							
+						elsif match = scan(/#{METHOD_NAME}/o)
+							if last_token_dot
+								type = if match[/^[A-Z]/] then :constant else :ident end
+							else
+								type = IDENT_KIND[match]
+								if type == :ident and match[/^[A-Z]/]
+									type = :constant
+								elsif type == :reserved
+									state = DEF_NEW_STATE[match]
+								end
+							end
+							fancy_allowed = regexp_allowed = REGEXP_ALLOWED[match]
+							
+						elsif match = scan(/ ['"] /mx)
+							tokens << [:open, :string]
+							type = :delimiter
+							state = StringState.new :string, match != '\'', match  # important for streaming
+							
+						elsif match = scan(/#{INSTANCE_VARIABLE}/o)
+							type = :instance_variable
+							
+						elsif regexp_allowed and match = scan(/ \/ /mx)
+							tokens << [:open, :regexp]
+							type = :delimiter
+							interpreted = true
+							state = StringState.new :regexp, interpreted, match
+							if parse_regexp
+								tokens = []
+								saved_tokens = tokens
+							end
+							
+						elsif match = scan(/#{NUMERIC}/o)
+							type = if match[/#{FLOAT}/o] then :float else :integer end
+
+						elsif fancy_allowed and match = scan(/#{SYMBOL}/o)
+							case match[1]
+							when ?', ?"
+								tokens << [:open, :symbol]
+								state = StringState.new :symbol, match[1] == ?", match[1,1]
+							end
+							type = :symbol
+							
+						elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
+							indented = self[1] == '-'
+							quote = self[3]
+							delim = self[quote ? 4 : 2]
+							type = QUOTE_TO_TYPE[quote]
+							tokens << [:open, type]
+							tokens << [match, :delimiter]
+							match = :close
+							heredoc = StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
+							heredocs ||= []  # create heredocs if empty
+							heredocs << heredoc
+							
+						elsif fancy_allowed and match = scan(/#{FANCY_START}/o)
+							type, interpreted = *FancyStringType.fetch(self[1]) do
+								raise 'Unknown fancy string: %%%p' % k, tokens
+							end
+							tokens << [:open, type]
+							state = StringState.new type, interpreted, self[2]
+							type = :delimiter
+
+						elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
+							type = :integer
+
+						elsif match = scan(/ [\/%<?:] /x)
+							regexp_allowed = fancy_allowed = :set
+							type = :operator
+
+						elsif match = scan(/`/)
+							if last_token_dot
+								type = :operator
+							else
+								tokens << [:open, :shell]
+								type = :delimiter
+								state = StringState.new :shell, true, match
+							end
+							
+						elsif match = scan(/#{GLOBAL_VARIABLE}/o)
+							type = :global_variable
+							
+						elsif match = scan(/#{CLASS_VARIABLE}/o)
+							type = :class_variable
+							
+						else
+							match = getch
+							
+						end
+						
+					elsif state == :def_expected
+						if match = scan(/ (?: #{VARIABLE} (?: ::#{IDENT} )* \. )? #{METHOD_NAME_EX} /ox)
+							type = :method
+						else
+							match = getch
+						end
+						state = :initial
+
+					elsif state == :module_expected
+						if match = scan(/<</)
+							type = :operator
+						else
+							if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
+								type = :class
+							else
+								match = getch
+							end
+						end
+						state = :initial
+						
+					end
+
+					regexp_allowed = regexp_allowed == :set
+					fancy_allowed = fancy_allowed == :set
+					last_token_dot = last_token_dot == :set
+
+					if $DEBUG
+						raise_inspect 'error token %p in line %d' % [tokens.last, line], tokens if not type or type == :error
+					end
+
+					tokens << [match, type]
+					
+					if last_state
+						state = last_state
+						last_state = nil
+					end
+# }}}
+				end
+			end
+
+			tokens
+		end
+	end
+
+end end
+# vim:fdm=marker