coderay 0.4.3.48

data/lib/coderay/scanner.rb

@@ -0,0 +1,197 @@
+module CodeRay
+	
+	require 'coderay/helpers/plugin'
+
+	# = Scanners
+	# 
+	# $Id: scanner.rb 39 2005-09-29 04:35:37Z murphy $
+	# 
+	# This module holds the Scanner class and its subclasses.
+	# For example, the Ruby scanner is named CodeRay::Scanners::Ruby
+	# can be found in coderay/scanners/ruby.
+	# 
+	# Scanner also provides methods and constants for the register
+	# mechanism and the [] method that returns the Scanner class
+	# belonging to the given lang.
+	#
+	# See PluginHost.
+	module Scanners
+		extend PluginHost
+		plugin_path 'coderay/scanners'
+
+		require 'strscan'
+
+		# = Scanner
+		#
+		# The base class for all Scanners.
+		#
+		# It is a subclass of Ruby's great +StringScanner+, which
+		# makes it easy to access the scanning methods inside.
+		#
+		# It is also +Enumerable+, so you can use it like an Array of
+		# Tokens:
+		# 
+		#   require 'coderay'
+		#   
+		#   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
+		#   
+		#   for text, kind in c_scanner
+		#     puts text if kind == :operator
+		#   end
+		#   
+		#   # prints: (*==)++;
+		# 
+		# OK, this is a very simple example :)
+		# You can also use +map+, +any?+, +find+ and even +sort_by+,
+		# if you want.
+		class Scanner < StringScanner
+			extend Plugin
+			plugin_host Scanners
+
+			# Raised if a Scanner fails while scanning
+			ScanError = Class.new(Exception)
+
+			require 'coderay/helpers/scanner_helper'
+
+			# The default options for all scanner classes.
+			# 
+			# Define @default_options for subclasses.
+			DEFAULT_OPTIONS = { :stream => false }
+
+			class << self
+
+				# Returns if the Scanner can be used in streaming mode.
+				def streamable?
+					is_a? Streamable
+				end
+
+			end
+
+=begin
+## Excluded for speed reasons; protected seems to make methods slow.
+
+# Save the StringScanner methods from being called.
+# This would not be useful for highlighting.
+	strscan_public_methods =
+		StringScanner.instance_methods -
+		StringScanner.ancestors[1].instance_methods
+	protected(*strscan_public_methods)
+=end
+
+			# Create a new Scanner.
+			#
+			# * +code+ is the input String and is handled by the superclass
+			#   StringScanner.
+			# * +options+ is a Hash with Symbols as keys.
+			#   It is merged with the default options of the class (you can
+			#   overwrite default options here.)
+			# * +block+ is the callback for streamed highlighting.
+			#
+			# If you set :stream to +true+ in the options, the Scanner uses a
+			# TokenStream with the +block+ as callback to handle the tokens.
+			#
+			# Else, a Tokens object is used.
+			def initialize code, options = {}, &block
+				@options = self.class::DEFAULT_OPTIONS.merge options
+				raise "I am only the basic Scanner class. I can't scan "\
+					"anything. :( Use my subclasses." if self.class == Scanner
+
+				# I love this hack. It seems to silence
+				# all dos/unix/mac newline problems.
+				super code.gsub(/\r\n?/, "\n")
+
+				if @options[:stream]
+					warn "warning in CodeRay::Scanner.new: :stream is set, "\
+						"but no block was given" unless block_given?
+					raise NotStreamableError, self unless kind_of? Streamable
+					@tokens = TokenStream.new(&block)
+				else
+					warn "warning in CodeRay::Scanner.new: Block given, "\
+						"but :stream is #{@options[:stream]}" if block_given?
+					@tokens = Tokens.new
+				end
+			end
+
+			# More mnemonic accessor name for the input string.
+			alias code string
+
+			# Scans the code and returns all tokens in a Tokens object.
+			def tokenize options = {}
+				options = @options.merge({}) #options
+				if @options[:stream]  # :stream must have been set already
+					reset ## what is this for?
+					scan_tokens @tokens, options
+					@tokens
+				else
+					@cached_tokens ||= scan_tokens @tokens, options
+				end
+			end
+
+			# You can also see tokenize as a read-only attribute
+			alias tokens tokenize
+
+			# Traverses the tokens.
+			def each &block
+				raise ArgumentError, 
+					'Cannot traverse TokenStream.' if @options[:stream]
+				tokens.each(&block)
+			end
+			include Enumerable
+
+			# The current line position of the scanner.
+			#
+			# Beware, this is implemented inefficiently. It should be used
+			# for debugging only.
+			def line
+				string[0..pos].count("\n") + 1
+			end
+
+		protected
+
+			# This is the central method, and commonly the only one a
+			# subclass implements.
+			# 
+			# Subclasses must implement this method; it must return +tokens+
+			# and must only use Tokens#<< for storing scanned tokens!
+			def scan_tokens tokens, options
+				raise NotImplementedError,
+					"#{self.class}#scan_tokens not implemented."
+			end
+
+			# Scanner error with additional status information
+			def raise_inspect msg, tokens, ambit = 30
+				raise ScanError, <<-EOE % [
+
+
+***ERROR in %s: %s
+
+tokens:
+%s
+
+current line: %d  pos = %d
+matched: %p
+bol? = %p,  eos? = %p
+
+surrounding code:
+%p  ~~  %p
+
+
+***ERROR***
+
+				EOE
+				File.basename(caller[0]),
+				msg,
+				tokens.last(10).map { |t| t.inspect }.join("\n"),
+				line, pos,
+				matched, bol?, eos?,
+				string[pos-ambit,ambit],
+				string[pos,ambit],
+				]
+			end
+
+		end
+
+	end
+end
+
+# vim:sw=2:ts=2:noet:tw=78

data/lib/coderay/scanners/c.rb

@@ -0,0 +1,147 @@
+module CodeRay module Scanners
+	
+	class C < Scanner
+
+		register_for :c
+		
+		RESERVED_WORDS = [
+			'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
+			'for', 'goto', 'if', 'return', 'switch', 'while',
+			'struct', 'union', 'enum', 'typedef',
+			'static', 'register', 'auto', 'extern',
+			'sizeof',
+			'volatile', 'const',  # C89
+			'inline', 'restrict', # C99			
+		]
+
+		PREDEFINED_TYPES = [
+			'int', 'long', 'short', 'char', 'void',
+			'signed', 'unsigned', 'float', 'double',
+			'bool', 'complex',  # C99
+		]
+
+		PREDEFINED_CONSTANTS = [
+			'EOF', 'NULL',
+			'true', 'false',  # C99
+		]
+
+		IDENT_KIND = Scanner::WordList.new(:ident).
+			add(RESERVED_WORDS, :reserved).
+			add(PREDEFINED_TYPES, :pre_type).
+			add(PREDEFINED_CONSTANTS, :pre_constant)
+
+		ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
+		UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
+
+		def scan_tokens tokens, options
+
+			state = :initial
+
+			until eos?
+
+				kind = :error
+				match = nil
+
+				if state == :initial
+					
+					if scan(/ \s+ | \\\n /x)
+						kind = :space
+						
+					elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
+						kind = :comment
+
+					elsif match = scan(/ \# \s* if \s* 0 /x)
+						match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
+						kind = :comment
+						
+					elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
+						kind = :operator
+						
+					elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+						kind = IDENT_KIND[match]
+						if kind == :ident and check(/:(?!:)/)
+							match << scan(/:/)
+							kind = :label
+						end
+						
+					elsif match = scan(/L?"/)
+						tokens << [:open, :string]
+						if match[0] == ?L
+							tokens << ['L', :modifier]
+							match = '"'
+						end
+						state = :string
+						kind = :delimiter
+						
+					elsif scan(/#\s*(\w*)/)
+						kind = :preprocessor  # FIXME multiline preprocs
+						state = :include_expected if self[1] == 'include'
+						
+					elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
+						kind = :char
+						
+					elsif scan(/0[xX][0-9A-Fa-f]+/)
+						kind = :hex
+						
+					elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
+						kind = :oct
+						
+					elsif scan(/(?:\d+)(?![.eEfF])/)
+						kind = :integer
+						
+					elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
+						kind = :float
+
+					else
+						getch
+					end
+					
+				elsif state == :string
+					if scan(/[^\\"]+/)
+						kind = :content
+					elsif scan(/"/)
+						tokens << ['"', :delimiter]
+						tokens << [:close, :string]
+						state = :initial
+						next
+					elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
+						kind = :char
+					elsif scan(/ \\ | $ /x)
+						kind = :error
+						state = :initial
+					else
+						raise "else case \" reached; %p not handled." % peek(1), tokens
+					end
+					
+				elsif state == :include_expected
+					if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+						kind = :include
+						state = :initial
+						
+					elsif match = scan(/\s+/)
+						kind = :space
+						state = :initial if match.index ?\n
+						
+					else
+						getch
+						
+					end
+					
+				else
+					raise 'else-case reached', tokens
+					
+				end
+				
+				match ||= matched
+				raise [match, kind], tokens if kind == :error
+
+				tokens << [match, kind]
+				
+			end
+			
+			tokens
+		end
+
+	end
+
+end end

data/lib/coderay/scanners/delphi.rb

@@ -0,0 +1,123 @@
+module CodeRay module Scanners
+	
+	class Delphi < Scanner
+
+		register_for :delphi
+		
+		RESERVED_WORDS = [
+			'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
+			'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
+			'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
+			'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in',
+			'inherited', 'initialization', 'inline', 'interface', 'is', 'label',
+			'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed',
+			'procedure', 'program', 'property', 'raise', 'record', 'repeat',
+			'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
+			'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
+			'xor', 'on'
+		]
+
+		DIRECTIVES = [
+			'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
+			'contains', 'deprecated', 'dispid', 'dynamic', 'export',
+			'external', 'far', 'forward', 'implements', 'local', 
+			'near', 'nodefault', 'on', 'overload', 'override',
+			'package', 'pascal', 'platform', 'private', 'protected', 'public',
+			'published', 'read', 'readonly', 'register', 'reintroduce',
+			'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
+			'virtual', 'write', 'writeonly'
+		]
+
+		IDENT_KIND = Scanner::WordList.new(:ident, :case_ignore).
+			add(RESERVED_WORDS, :reserved).
+			add(DIRECTIVES, :directive)
+
+		def scan_tokens tokens, options
+
+			state = :initial
+
+			until eos?
+
+				kind = :error
+				match = nil
+
+				if state == :initial
+					
+					if scan(/ \s+ /x)
+						kind = :space
+						
+					elsif scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
+						kind = :preprocessor
+						
+					elsif scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
+						kind = :comment
+						
+					elsif scan(/ [-+*\/=<>:;,.@\^|\(\)\[\]]+ /x)
+						kind = :operator
+						
+					elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
+						kind = IDENT_KIND[match]
+						
+					elsif match = scan(/ ' ( [^\n']|'' ) (?:'|$) /x)
+						tokens << [:open, :char]
+						tokens << ["'", :delimiter]
+						tokens << [self[1], :content]
+						tokens << ["'", :delimiter]
+						tokens << [:close, :char]
+						next
+						
+					elsif match = scan(/ ' /x)
+						tokens << [:open, :string]
+						state = :string
+						kind = :delimiter
+						
+					elsif scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
+						kind = :char
+						
+					elsif scan(/ \$ [0-9A-Fa-f]+ /x)
+						kind = :hex
+						
+					elsif scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
+						kind = :integer
+						
+					elsif scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
+						kind = :float
+
+					else
+						getch
+					end
+					
+				elsif state == :string
+					if scan(/[^\n']+/)
+						kind = :content
+					elsif scan(/''/)
+						kind = :char
+					elsif scan(/'/)
+						tokens << ["'", :delimiter]
+						tokens << [:close, :string]
+						state = :initial
+						next
+					elsif scan(/\n/)
+						state = :initial
+					else
+						raise "else case \' reached; %p not handled." % peek(1), tokens
+					end
+					
+				else
+					raise 'else-case reached', tokens
+					
+				end
+				
+				match ||= matched
+				raise [match, kind], tokens if kind == :error
+
+				tokens << [match, kind]
+				
+			end
+			
+			tokens
+		end
+
+	end
+
+end end