coderay 0.4.3.48

data/lib/coderay/tokens.rb

@@ -0,0 +1,324 @@
+module CodeRay
+
+	# = Tokens
+	#
+	# The Tokens class represents a list of tokens returnd from
+	# a Scanner.
+	#
+	# A token is not a special object, just a two-element Array
+	# consisting of
+	# * the _token_ _kind_ (a Symbol representing the type of the token)
+	# * the _token_ _text_ (the original source of the token in a String)
+	#
+	# A token looks like this:
+	# 
+	#   [:comment, '# It looks like this']
+	#   [:float, '3.1415926']
+	#   [:error, '���']
+	# 
+	# Some scanners also yield some kind of sub-tokens, represented by special
+	# token texts, namely :open and :close .
+	# 
+	# The Ruby scanner, for example, splits "a string" into:
+	# 
+	#  [
+	#   [:open, :string],
+	#   [:delimiter, '"'],
+	#   [:content, 'a string'],
+	#   [:delimiter, '"'],
+	#   [:close, :string]
+	#  ]
+	# 
+	# Tokens is also the interface between Scanners and Encoders:
+	# The input is split and saved into a Tokens object. The Encoder
+	# then builds the output from this object.
+	# 
+	# Thus, the syntax below becomes clear:
+	#
+	#   CodeRay.scan('price = 2.59', :ruby).html
+	#   # the Tokens object is here -------^
+	# 
+	# See how small it is? ;)
+	# 
+	# Tokens gives you the power to handle pre-scanned code very easily:
+	# You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
+	# that you put in your DB.
+	#
+	# Tokens' subclass TokenStream allows streaming to save memory.
+	class Tokens < Array
+
+		class << self
+
+			# Convert the token to a string.
+			# 
+			# This format is used by Encoders.Tokens.
+			# It can be reverted using read_token.
+			def write_token text, type
+				if text.is_a? String
+					"#{type}\t#{escape(text)}\n"
+				else
+					":#{text}\t#{type}\t\n"
+				end
+			end
+
+			# Read a token from the string.
+			# 
+			# Inversion of write_token.
+			#
+			# TODO Test this!
+			def read_token token
+				type, text = token.split("\t", 2)
+				if type[0] == ?:
+					[text.to_sym, type[1..-1].to_sym]
+				else
+					[type.to_sym, unescape(text)]
+				end
+			end
+
+			# Escapes a string for use in write_token.
+			def escape text
+				text.gsub(/[\n\\]/, '\\\\\&')
+			end
+
+			# Unescapes a string created by escape.
+			def unescape text
+				text.gsub(/\\[\n\\]/) { |m| m[1,1] }
+			end
+
+		end
+
+		# Whether the object is a TokenStream.
+		#
+		# Returns false.
+		def stream?
+			false
+		end
+
+		alias :orig_each :each
+		# Iterates over all tokens.
+		# 
+		# If a filter is given, only tokens of that kind are yielded.
+		def each kind_filter = nil, &block
+			unless kind_filter
+				orig_each(&block)
+			else
+				orig_each do |text, kind|
+					next unless kind == kind_filter
+					yield text, kind
+				end
+			end
+		end
+
+		# Iterates over all text tokens.
+		# Range tokens like [:open, :string] are left out.
+		#
+		# Example:
+		#   tokens.each_text_token { |text, kind| text.replace html_escape(text) }
+		def each_text_token
+			orig_each do |text, kind|
+				next unless text.respond_to? :to_str
+				yield text, kind
+			end
+		end
+
+		# Encode the tokens using encoder.
+		#
+		# encoder can be
+		# * a symbol like :html oder :statistic
+		# * an Encoder class
+		# * an Encoder object
+		# 
+		# options are passed to the encoder.
+		def encode encoder, options = {}
+			unless encoder.is_a? Encoders::Encoder
+				unless encoder.is_a? Class
+					encoder_class = Encoders[encoder]
+				end
+				encoder = encoder_class.new options
+			end
+			encoder.encode_tokens self, options
+		end
+
+
+		# Turn into a string using Encoders::Text.
+		#
+		# +options+ are passed to the encoder if given.
+		def to_s options = {}
+			encode :text, options
+		end
+
+
+		# Redirects unknown methods to encoder calls.
+		#
+		# For example, if you call +tokens.html+, the HTML encoder
+		# is used to highlight the tokens.
+		def method_missing meth, options = {}
+			Encoders[meth].new(options).encode_tokens self
+		end
+
+		# Returns the tokens compressed by joining consecutive
+		# tokens of the same kind.
+		# 
+		# This can not be undone, but should yield the same output
+		# in most Encoders.  It basically makes the output smaller.
+		#
+		# Combined with dump, it saves space for the cost
+		# calculating time.
+		#
+		# If the scanner is written carefully, this is not required - 
+		# for example, consecutive //-comment lines can already be 
+		# joined in one token by the Scanner.
+		def optimize
+			print ' Tokens#optimize: before: %d - ' % size if $DEBUG
+			last_kind = last_text = nil
+			new = self.class.new
+			each do |text, kind|
+				if text.is_a? String
+					if kind == last_kind
+						last_text << text
+					else
+						new << [last_text, last_kind] if last_kind
+						last_text = text
+						last_kind = kind
+					end
+				else
+					new << [last_text, last_kind] if last_kind
+					last_kind = last_text = nil
+					new << [text, kind]
+				end
+			end
+			new << [last_text, last_kind] if last_kind
+			print 'after: %d (%d saved = %2.0f%%)' % 
+				[new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
+			new
+		end
+
+		# Compact the object itself; see optimize.
+		def optimize!
+			replace optimize
+		end
+
+		# Dumps the object into a String that can be saved
+		# in files or databases.
+		#
+		# The dump is created with Marshal.dump;
+		# In addition, it is gzipped using GZip.gzip.
+		#
+		# The returned String object includes Undumping
+		# so it has an #undump method. See Tokens.load.
+		#
+		# You can configure the level of compression,
+		# but the default value 7 should be what you want
+		# in most cases as it is a good comprimise between
+		# speed and compression rate.
+		# 
+		# See GZip module.
+		def dump gzip_level = 7
+			require 'coderay/helpers/gzip_simple'
+			dump = Marshal.dump self
+			dump = dump.gzip gzip_level
+			dump.extend Undumping
+		end
+
+		# The total size of the tokens;
+		# Should be equal to the input size before
+		# scanning.
+		def text_size
+			map { |t, k| t }.join.size
+		end
+
+		# Include this module to give an object an #undump
+		# method.
+		#
+		# The string returned by Tokens.dump includes Undumping.
+		module Undumping
+			# Calls Tokens.load with itself.
+			def undump
+				Tokens.load self
+			end
+		end
+
+		# Undump the object using Marshal.load, then
+		# unzip it using GZip.gunzip.
+		# 
+		# The result is commonly a Tokens object, but
+		# this is not guaranteed.
+		def Tokens.load dump
+			require 'coderay/helpers/gzip_simple'
+			dump = dump.gunzip
+			@dump = Marshal.load dump
+		end
+
+	end
+
+
+	# = TokenStream
+	#
+	# The TokenStream class is a fake Array without elements.
+	# 
+	# It redirects the method << to a block given at creation.
+	#
+	# This allows scanners and Encoders to use streaming (no
+	# tokens are saved, the input is highlighted the same time it
+	# is scanned) with the same code.
+	#
+	# See CodeRay.encode_stream and CodeRay.scan_stream
+	class TokenStream < Tokens
+
+		# Whether the object is a TokenStream.
+		#
+		# Returns true.
+		def stream?
+			true
+		end
+
+		# The Array is empty, but size counts the tokens given by <<.
+		attr_reader :size
+
+		# Creates a new TokenStream that calls +block+ whenever
+		# its << method is called.
+		#
+		# Example:
+		#
+		#   require 'coderay'
+		# 	
+		#   token_stream = CodeRay::TokenStream.new do |kind, text|
+		#     puts 'kind: %s, text size: %d.' % [kind, text.size]
+		#   end
+		#   
+		#   token_stream << [:regexp, '/\d+/']
+		#   #-> kind: rexpexp, text size: 5.
+		#
+		def initialize &block
+			raise ArgumentError, 'Block expected for streaming.' unless block
+			@callback = block
+			@size = 0
+		end
+
+		# Calls +block+ with +token+ and increments size.
+		#
+		# Returns self.
+		def << token
+			@callback.call token
+			@size += 1
+			self
+		end
+
+		# This method is not implemented due to speed reasons. Use Tokens.
+		def text_size
+			raise NotImplementedError, 'This method is not implemented due to speed reasons.'
+		end
+
+		# A TokenStream cannot be dumped. Use Tokens.
+		def dump
+			raise NotImplementedError, 'A TokenStream cannot be dumped.'
+		end
+
+		# A TokenStream cannot be optimized. Use Tokens.
+		def optimize
+			raise NotImplementedError, 'A TokenStream cannot be optimized.'
+		end
+
+	end
+
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.10
+specification_version: 1
+name: coderay
+version: !ruby/object:Gem::Version 
+  version: 0.4.3.48
+date: 2005-10-01
+summary: CodeRay is a fast syntax highlighter engine for many languages.
+require_paths: 
+  - lib
+email: murphy@cYcnus.de
+homepage: http://rd.cycnus.de/coderay
+rubyforge_project: coderay
+description: "CodeRay is a Ruby library for syntax highlighting. I try to make CodeRay easy to
+  use and intuitive, but at the same time fully featured, complete, fast and
+  efficient.  Usage is simple: require 'coderay' code = 'some %q(weird (Ruby)
+  can't shock) me!' puts CodeRay.scan(code, :ruby).html"
+autorequire: coderay
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - murphy
+files: 
+  - "./lib/coderay.rb"
+  - "./lib/coderay/encoder.rb"
+  - "./lib/coderay/scanner.rb"
+  - "./lib/coderay/tokens.rb"
+  - "./lib/coderay/encoders/yaml.rb"
+  - "./lib/coderay/encoders/div.rb"
+  - "./lib/coderay/encoders/tokens.rb"
+  - "./lib/coderay/encoders/statistic.rb"
+  - "./lib/coderay/encoders/null.rb"
+  - "./lib/coderay/encoders/span.rb"
+  - "./lib/coderay/encoders/html.rb"
+  - "./lib/coderay/encoders/text.rb"
+  - "./lib/coderay/encoders/count.rb"
+  - "./lib/coderay/encoders/helpers/html_helper.rb"
+  - "./lib/coderay/encoders/helpers/html_output.rb"
+  - "./lib/coderay/encoders/helpers/html_css.rb"
+  - "./lib/coderay/helpers/plugin.rb"
+  - "./lib/coderay/helpers/filetype.rb"
+  - "./lib/coderay/helpers/gzip_simple.rb"
+  - "./lib/coderay/helpers/scanner_helper.rb"
+  - "./lib/coderay/scanners/c.rb"
+  - "./lib/coderay/scanners/delphi.rb"
+  - "./lib/coderay/scanners/plaintext.rb"
+  - "./lib/coderay/scanners/ruby.rb"
+  - "./lib/coderay/scanners/helpers/ruby_helper.rb"
+  - "./demo/demo_global_vars2.rb"
+  - "./demo/demo_encoder.rb"
+  - "./demo/demo_div.rb"
+  - "./demo/demo_count.rb"
+  - "./demo/demo_simple.rb"
+  - "./demo/demo_tokens.rb"
+  - "./demo/demo_css.rb"
+  - "./demo/demo_html.rb"
+  - "./demo/demo_dump.rb"
+  - "./demo/demo_stream2.rb"
+  - "./demo/demo_load_encoder.rb"
+  - "./demo/demo_html2.rb"
+  - "./demo/demo_scanner.rb"
+  - "./demo/demo_stream.rb"
+  - "./demo/demo_global_vars.rb"
+  - "./demo/demo_more.rb"
+  - "./demo/demo_server.rb"
+  - "./README"
+  - "./LICENSE"
+test_files: []
+rdoc_options: 
+  - "-SNw2"
+  - "-mREADME"
+  - "-a"
+  - "-t CodeRay Documentation"
+extra_rdoc_files: 
+  - "./README"
+executables: []
+extensions: []
+requirements: 
+  - strscan
+dependencies: []