coderay 0.4.3.48

data/demo/demo_html2.rb

@@ -0,0 +1,11 @@
+require 'coderay'
+
+# scan this file
+tokens = CodeRay.scan(File.read($0) * 1, :ruby)
+
+# output it with two styles of line numbers
+out = tokens.div(:line_numbers => :table)
+out << '<hr />'
+out << tokens.div(:line_numbers => :inline, :line_number_start => 8)
+
+puts out.page

data/demo/demo_load_encoder.rb

@@ -0,0 +1,17 @@
+require 'coderay'
+
+begin
+	CodeRay::Encoders::YAML
+rescue
+	puts 'CodeRay::Encoders::YAML is not defined; you must load it first.'
+end
+
+yaml_encoder = CodeRay::Encoders[:yaml] 
+print 'Now it is loaded: '
+p yaml_encoder
+puts 'See?'
+
+tokens_encoder = require_plugin 'CodeRay::Encoders/tokens'
+print 'Require is also possible: '
+p tokens_encoder
+puts 'See?'

data/demo/demo_more.rb

@@ -0,0 +1,204 @@
+require 'coderay'
+
+c, ruby = DATA.read.split(/^---$/)
+DATA.rewind
+me = DATA.read[/.*^__END__$/m]
+$input = c + ruby + me
+
+require 'benchmark'
+time = Benchmark.realtime do
+
+	# here CodeRay comes to play
+	hl = CodeRay.encoder(:html, :tab_width => 2, :line_numbers => :table, :wrap => :div)
+	c = hl.highlight c, :c
+	ruby = hl.highlight ruby, :ruby
+	me = hl.highlight me, :ruby
+
+	body = %w[C Ruby Genereated\ by].zip([c, ruby, me]).map do |title, code|
+		"<h1>#{title}</h1>\n#{code}"
+	end.join
+	body = hl.class::Output.new(body, :div).page!
+
+	# CodeRay also provides a simple page generator
+	$output = body #hl.class.wrap_in_page body
+end
+
+File.open('test.html', 'w') do |f|
+	f.write $output
+end
+puts 'Input: %dB, Output: %dB' % [$input.size, $output.size]
+puts 'Created "test.html" in %0.3f seconds (%d KB/s). Take a look with your browser.' % [time, $input.size / 1024.0 / time]
+
+__END__
+/**********************************************************************
+
+  version.c -
+
+  $Author: nobu $
+  $Date: 2004/03/25 12:01:40 $
+  created at: Thu Sep 30 20:08:01 JST 1993
+
+  Copyright (C) 1993-2003 Yukihiro Matsumoto
+
+**********************************************************************/
+
+#include "ruby.h"
+#include "version.h"
+#include <stdio.h>
+
+const char ruby_version[] = RUBY_VERSION;
+const char ruby_release_date[] = RUBY_RELEASE_DATE;
+const char ruby_platform[] = RUBY_PLATFORM;
+
+void
+Init_version()
+{
+    VALUE v = rb_obj_freeze(rb_str_new2(ruby_version));
+    VALUE d = rb_obj_freeze(rb_str_new2(ruby_release_date));
+    VALUE p = rb_obj_freeze(rb_str_new2(ruby_platform));
+
+    rb_define_global_const("RUBY_VERSION", v);
+    rb_define_global_const("RUBY_RELEASE_DATE", d);
+    rb_define_global_const("RUBY_PLATFORM", p);
+}
+
+void
+ruby_show_version()
+{
+    printf("ruby %s (%s) [%s]\n", RUBY_VERSION, RUBY_RELEASE_DATE, RUBY_PLATFORM);
+}
+
+void
+ruby_show_copyright()
+{
+    printf("ruby - Copyright (C) 1993-%d Yukihiro Matsumoto\n", RUBY_RELEASE_YEAR);
+    exit(0);
+}
+---
+#
+# = ostruct.rb: OpenStruct implementation
+#
+# Author:: Yukihiro Matsumoto
+# Documentation:: Gavin Sinclair
+#
+# OpenStruct allows the creation of data objects with arbitrary attributes.
+# See OpenStruct for an example.
+#
+
+#
+# OpenStruct allows you to create data objects and set arbitrary attributes.
+# For example:
+#
+#   require 'ostruct' 
+#
+#   record = OpenStruct.new
+#   record.name    = "John Smith"
+#   record.age     = 70
+#   record.pension = 300
+#   
+#   puts record.name     # -> "John Smith"
+#   puts record.address  # -> nil
+#
+# It is like a hash with a different way to access the data.  In fact, it is
+# implemented with a hash, and you can initialize it with one.
+#
+#   hash = { "country" => "Australia", :population => 20_000_000 }
+#   data = OpenStruct.new(hash)
+#
+#   p data        # -> <OpenStruct country="Australia" population=20000000>
+#
+class OpenStruct
+  #
+  # Create a new OpenStruct object.  The optional +hash+, if given, will
+  # generate attributes and values.  For example.
+  #
+  #   require 'ostruct'
+  #   hash = { "country" => "Australia", :population => 20_000_000 }
+  #   data = OpenStruct.new(hash)
+  #
+  #   p data        # -> <OpenStruct country="Australia" population=20000000>
+  #
+  # By default, the resulting OpenStruct object will have no attributes. 
+  #
+  def initialize(hash=nil)
+    @table = {}
+    if hash
+      for k,v in hash
+	@table[k.to_sym] = v
+        new_ostruct_member(k)
+      end
+    end
+  end
+
+  # Duplicate an OpenStruct object members. 
+  def initialize_copy(orig)
+    super
+    @table = @table.dup
+  end
+
+  def marshal_dump
+    @table
+  end
+  def marshal_load(x)
+    @table = x
+    @table.each_key{|key| new_ostruct_member(key)}
+  end
+
+  def new_ostruct_member(name)
+    unless self.respond_to?(name)
+      self.instance_eval %{
+        def #{name}; @table[:#{name}]; end
+        def #{name}=(x); @table[:#{name}] = x; end
+      }
+    end
+  end
+
+  def method_missing(mid, *args) # :nodoc:
+    mname = mid.id2name
+    len = args.length
+    if mname =~ /=$/
+      if len != 1
+	raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1)
+      end
+      if self.frozen?
+	raise TypeError, "can't modify frozen #{self.class}", caller(1)
+      end
+      mname.chop!
+      @table[mname.intern] = args[0]
+      self.new_ostruct_member(mname)
+    elsif len == 0
+      @table[mid]
+    else
+      raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1)
+    end
+  end
+
+  #
+  # Remove the named field from the object.
+  #
+  def delete_field(name)
+    @table.delete name.to_sym
+  end
+
+  #
+  # Returns a string containing a detailed summary of the keys and values.
+  #
+  def inspect
+    str = "<#{self.class}"
+    for k,v in @table
+      str << " #{k}=#{v.inspect}"
+    end
+    str << ">"
+  end
+
+  attr_reader :table # :nodoc:
+  protected :table
+
+  #
+  # Compare this object and +other+ for equality.
+  #
+  def ==(other)
+    return false unless(other.kind_of?(OpenStruct))
+    return @table == other.table
+  end
+end

data/demo/demo_scanner.rb

@@ -0,0 +1,36 @@
+require 'coderay'
+
+c_code = "if (*p == '{') nest++;"
+puts 'C Code: ' + c_code
+puts
+
+c_scanner = CodeRay::Scanners[:c].new c_code
+
+puts '> print only operators:'
+for text, kind in c_scanner
+	print text if kind == :operator
+end
+puts
+puts '-' * 30
+puts
+
+ruby_code = %q!ruby_code(:can, BE, %r[q[ui]te #{ /comple/x },] => $-s, &?\xee)!
+puts 'Ruby Code: ' + ruby_code
+puts
+
+ruby_scanner = CodeRay::Scanners[:ruby].new ruby_code
+
+puts '> has a string?'
+puts ruby_scanner.
+	any? { |text, kind| kind == :string }
+puts
+
+puts '> number of regexps?'
+puts ruby_scanner.
+	select { |token| token == [:open, :regexp] }.size
+puts
+
+puts '> has a string?'
+puts ruby_scanner.
+	reject { |text, kind| not text.is_a? String }.
+	map { |text, kind| %("#{text}" (#{kind})) }.join(', ')

data/demo/demo_server.rb

@@ -0,0 +1,92 @@
+# CodeRay dynamic highlighter
+#
+# Usage: start this and your browser.
+# 
+# Go to http://localhost:49374/?<path to the file>
+# (mnemonic: 49374 = Four-Nine-Three-Seven-Four = For No Token Shall Fall)
+# and you should get the highlighted version.
+
+require 'webrick'
+require 'pathname'
+
+class << File
+	alias dir? directory?
+end
+
+require 'erb'
+include ERB::Util
+def url_decode s
+	s.to_s.gsub(/%([0-9a-f]{2})/i) { [$1.hex].pack 'C' }
+end
+
+class String
+	def to_link name = File.basename(self)
+		"<a href=\"?path=#{url_encode self}\">#{name}</a>"
+	end
+end
+
+require 'coderay'
+class CodeRayServlet < WEBrick::HTTPServlet::AbstractServlet
+
+	STYLE = 'style="font-family: sans-serif; color: navy;"'
+	BANNER = '<p><img src="http://rd.cYcnus.de/coderay/coderay-banner" style="border: 0" alt="HIghlighted by CodeRay"/></p>'
+
+	def do_GET req, res
+		q = req.query_string || ''
+		args = Hash[*q.scan(/(.*?)=(.*?)(?:&|$)/).flatten].each_value { |v| v.replace url_decode(v) }
+		path = args.fetch 'path', '.'
+		
+		backlinks = '<p>current path: %s<br />' % html_escape(path) +
+			(Pathname.new(path) + '..').cleanpath.to_s.to_link('up') + ' - ' +
+			'.'.to_link('current') + '</p>'
+		
+		res.body = 
+			if File.dir? path
+				path = Pathname.new(path).cleanpath.to_s
+				dirs, files = Dir[File.join(path, '*')].sort.partition { |p| File.dir? p }
+
+				page = "<html><head></head><body #{STYLE}>"
+				page << backlinks
+				
+				page << '<dl>'
+				page << "<dt>Directories</dt>\n" + dirs.map do |p|
+					"<dd>#{p.to_link}</dd>\n"
+				end.join << "\n"
+				page << "<dt>Files</dt>\n" + files.map do |p|
+					"<dd>#{p.to_link}</dd>\n"
+				end.join << "\n"
+				page << "</dl>\n"
+				page << "#{BANNER}</body></html>"
+			
+			elsif File.exist? path
+				div = CodeRay.scan_file(path).html :tab_width => 8, :wrap => :div
+				div.replace <<-DIV
+	<div #{STYLE}>
+		#{backlinks}
+#{div}
+	</div>
+	#{BANNER}
+				DIV
+				div.page
+			end
+
+		res['Content-Type'] = 'text/html'
+	end
+end
+
+# this is taken by "qip_msgd" - I don't know that.
+module CodeRay
+	PORT = 0xC0DE / 20
+end
+
+server = WEBrick::HTTPServer.new :Port => CodeRay::PORT
+
+server.mount '/', CodeRayServlet
+
+server.mount_proc '/version' do |req, res|
+	res.body = 'CodeRay::Version = ' + CodeRay::Version
+	res['Content-Type'] = "text/plain"
+end
+
+trap("INT") { server.shutdown }
+server.start

data/demo/demo_simple.rb

@@ -0,0 +1,10 @@
+
+# Load CodeRay
+# If this doesn't work, try ruby -rubygems.
+require 'coderay'
+
+# Generate HTML page for Ruby code.
+page = CodeRay.scan("puts 'Hello, world!'", :ruby).span
+
+# Print it
+puts page

data/demo/demo_stream.rb

@@ -0,0 +1,25 @@
+require 'coderay'
+
+code = File.read($0) * 500
+puts "Size of code: %d KB" % [code.size / 1024]
+
+puts "Use your system's memory tracker to see how much RAM this takes."
+print 'Press some key to continue...'; gets
+
+require 'benchmark'
+e = CodeRay.encoder(:div)
+for do_stream in [true, false]
+	puts "Scanning and encoding in %s mode, please wait..." %
+		[do_stream ? 'streaming' : 'normal']
+	output = ''
+	time = Benchmark.realtime do
+		if do_stream
+			output = e.encode_stream(code, :ruby)
+		else
+			output = e.encode_tokens(t = CodeRay.scan(code, :ruby))
+		end
+	end
+	puts 'Finished after %4.2f seconds.' % time
+	puts "Size of output: %d KB" % [output.size / 1024]
+	print 'Press some key to continue...'; gets
+end

data/demo/demo_stream2.rb

@@ -0,0 +1,8 @@
+require 'coderay'
+
+token_stream = CodeRay::TokenStream.new do |kind, text|
+  puts 'kind: %s, text size: %d.' % [kind, text.size]
+end
+
+token_stream << [:regexp, '/\d+/'] << [:space, "\n"]
+#-> kind: rexpexp, text size: 5.

data/demo/demo_tokens.rb

@@ -0,0 +1,3 @@
+require 'coderay'
+
+puts CodeRay.scan("puts 3 + 4, '3 + 4'", :ruby).tokens

data/lib/coderay.rb

@@ -0,0 +1,284 @@
+# = CodeRay Library
+#
+# $Id: coderay.rb 44 2005-10-01 00:59:05Z murphy $
+#
+# CodeRay is a Ruby library for syntax highlighting.
+#
+# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
+# fast and efficient.
+# 
+# See README.
+# 
+# It consists mainly of
+# * the main engine: CodeRay (Scanners::Scanner, Tokens/TokenStream, Encoders::Encoder), PluginHost
+# * the scanners in CodeRay::Scanners
+# * the encoders in CodeRay::Encoders
+# 
+# Here's a fancy graphic to light up this gray docu:
+# 
+# http://rd.cYcnus.de/coderay/scheme.png
+# 
+# == Documentation
+#
+# See CodeRay, Encoders, Scanners, Tokens.
+#
+# == Usage
+#
+# Remember you need RubyGems to use CodeRay. Run Ruby with -rubygems option
+# if required.
+#
+# === Highlight Ruby code in a string as html
+# 
+#   require 'coderay'
+#   print CodeRay.scan('puts "Hello, world!"', :ruby).html
+#
+#   # prints something like this:
+#   puts <span class="s">&quot;Hello, world!&quot;</span>
+# 
+# 
+# === Highlight C code from a file in a html div
+# 
+#   require 'coderay'
+#   print CodeRay.scan(File.read('ruby.h'), :c).div
+#   print CodeRay.scan_file('ruby.h').html.div
+# 
+# You can include this div in your page. The used CSS styles can be printed with
+# 
+#   % ruby -rcoderay -e "print CodeRay::Encoders[:html]::CSS"
+# 
+# === Highlight without typing too much
+#
+# If you are one of the hasty (or lazy, or extremely curious) people, just run this file:
+#
+#   % ruby -rubygems coderay.rb
+# 
+# If the output was to fast for you, try
+# 
+#   % ruby -rubygems coderay.rb > example.html
+#
+# and look at the file it created.
+# 
+# = CodeRay Module
+#
+# The CodeRay module provides convenience methods for the engine.
+#
+# * The +lang+ and +format+ arguments select Scanner and Encoder to use. These are
+#   simply lower-case symbols, like <tt>:python</tt> or <tt>:html</tt>.
+# * All methods take an optional hash as last parameter, +options+, that is send to
+#   the Encoder / Scanner.
+# * Input and language are always sorted in this order: +code+, +lang+.
+# 	(This is in alphabetical order, if you need a mnemonic ;)
+# 
+# You should be able to highlight everything you want just using this methods;
+# so there is no need to dive into CodeRay's deep class hierarchy.
+#
+# The exmaples in the demo/ directory demonstrate common cases using this interface.
+#	
+# = Basic Access Ways
+#
+# Read this to get a general view what CodeRay provides.
+# 
+# == Scanning
+#	
+#	Scanning means analysing an input string, splitting it up into Tokens.
+#	Each Token knows about what type it is: string, comment, class name, etc.
+#
+#	Each +lang+ (language) has its own Scanner; for example, <tt>:ruby</tt> code is
+#	handled by CodeRay::Scanners::RubyScanner.
+# 
+# CodeRay.scan:: Scan a string in a given language into Tokens.
+#                This is the most common method to use.
+# CodeRay.scan_file:: Scan a file and guess the language using FileType.
+# 
+# The Tokens object you get from these methods can encode itself; see Tokens.
+# 
+# == Encoding
+#
+# Encoding means compiling Tokens into an output. This can be colored HTML or
+# LaTeX, a textual statistic or just the number of non-whitespace tokens.
+# 
+# Each Encoder provides output in a specific +format+, so you select Encoders via
+# formats like <tt>:html</tt> or <tt>:statistic</tt>.
+# 
+# CodeRay.encode:: Scan and encode a string in a given language.
+# CodeRay.encode_tokens:: Encode the given tokens.
+# CodeRay.encode_file:: Scan a file, guess the language using FileType and encode it.
+#
+# == Streaming
+#
+# Streaming saves RAM by running Scanner and Encoder in some sort of
+# pipe mode; see TokenStream.
+# 
+# CodeRay.scan_stream:: Scan in stream mode.
+# 
+#	== All-in-One Encoding
+#
+# CodeRay.encode:: Highlight a string with a given input and output format.
+#
+# == Instanciating
+#	
+#	You can use an Encoder instance to highlight multiple inputs. This way, the setup
+#	for this Encoder must only be done once.
+#	
+# CodeRay.encoder:: Create an Encoder instance with format and options.
+#
+# There is no CodeRay.scanner method because Scanners are bound to an input string
+# on creation; you can't re-use them with another string.
+#
+#	The scanning methods provide more flexibility; we recommend to use these.
+module CodeRay
+	
+	Version = '0.4.4'
+	
+	require 'coderay/tokens'
+	require 'coderay/scanner'
+	require 'coderay/encoder'
+
+
+	class << self
+
+		# Scans the given +code+ (a String) with the Scanner for +lang+.
+		# 
+		# This is a simple way to use CodeRay. Example:
+		#  require 'coderay'
+		#  page = CodeRay.scan("puts 'Hello, world!'", :ruby).html
+		#
+		# See also demo/demo_simple.
+		def scan code, lang, options = {}, &block
+			scanner = Scanners[lang].new code, options, &block
+			scanner.tokenize
+		end
+
+		# Scans +filename+ (a path to a code file) with the Scanner for +lang+.
+		# 
+		# If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
+		# determine it. If it cannot find out what type it is, it uses
+		# CodeRay::Scanners::Plaintext.
+		#
+		# Calls CodeRay.scan.
+		# 
+		# Example:
+		#  require 'coderay'
+		#  page = CodeRay.scan_file('some_c_code.c').html
+		def scan_file filename, lang = :auto, options = {}, &block
+			file = IO.read filename
+			if lang == :auto
+				require 'coderay/helpers/filetype'
+				lang = FileType.fetch filename, :plaintext, true
+			end
+			scan file, lang, options = {}, &block
+		end
+
+		# Scan the +code+ (a string) with the scanner for +lang+.
+		# 
+		# Calls scan.
+		# 
+		# See CodeRay.scan.
+		def scan_stream code, lang, options = {}, &block
+			options[:stream] = true
+			scan code, lang, options, &block
+		end
+
+		# Encode a string in Streaming mode.
+		# 
+		# This starts scanning +code+ with the the Scanner for +lang+ 
+		# while encodes the output with the Encoder for +format+.
+		# +options+ will be passed to the Encoder.
+		#
+		# See CodeRay::Encoder.encode_stream
+		def encode_stream code, lang, format, options = {}
+			encoder(format, options).encode_stream code, lang, options
+		end
+
+		# Encode a string.
+		# 
+		# This scans +code+ with the the Scanner for +lang+ and then
+		# encodes it with the Encoder for +format+.
+		# +options+ will be passed to the Encoder.
+		#
+		# See CodeRay::Encoder.encode
+		def encode code, lang, format, options = {}
+			encoder(format, options).encode code, lang, options
+		end
+		
+		# Encode pre-scanned Tokens.
+		# Use this together with CodeRay.scan:
+		# 
+		#  require 'coderay'
+		#  
+		#  # Highlight a short Ruby code example in a HTML span
+		#  tokens = CodeRay.scan '1 + 2', :ruby
+		#  puts CodeRay.encode_tokens(tokens, :span)
+		#
+		def encode_tokens tokens, format, options = {}
+			encoder(format, options).encode_tokens tokens, options
+		end
+
+		# Encodes +filename+ (a path to a code file) with the Scanner for +lang+.
+		# 
+		# See CodeRay.scan_file.
+		# Notice that the second argument is the output +format+, not the input language.
+		# 
+		# Example:
+		#  require 'coderay'
+		#  page = CodeRay.encode_file 'some_c_code.c', :html
+		def encode_file filename, format, options = {}
+			tokens = scan_file filename, auto, get_scanner_options(options)
+			encode_tokens tokens, format, options
+		end
+
+		# Finds the Encoder class for +format+ and creates an instance, passing
+		# +options+ to it.
+		# 
+		# Example:
+		#  require 'coderay'
+		#  
+		#  stats = CodeRay.encoder(:statistic)
+		#  stats.encode("puts 17 + 4\n", :ruby)
+		#  
+		#  puts '%d out of %d tokens have the kind :integer.' % [
+		#  	stats.type_stats[:integer].count,
+		#  	stats.real_token_count
+		#  ]
+		#  #-> 2 out of 4 tokens have the kind :integer.
+		def encoder format, options = {}
+			Encoders[format].new options
+		end
+
+		# Extract the options for the scanner from the +options+ hash.
+		#
+		#	Returns an empty Hash if <tt>:scanner_options</tt> is not set.
+		#	
+		# This is used if a method like CodeRay.encode has to provide options
+		# for Encoder _and_ scanner.
+		def get_scanner_options options
+			options.fetch :scanner_options, {}
+		end
+
+	end
+
+	# This Exception is raised when you try to stream with something that is not
+	# capable of streaming.
+	class NotStreamableError < Exception
+		def initialize obj
+			@obj = obj
+		end
+
+		def to_s
+			'%s is not Streamable!' % @obj.class
+		end
+	end
+	
+	# A dummy module that is included by subclasses of CodeRay::Scanner an CodeRay::Encoder
+	# to show that they are able to handle streams.
+	module Streamable
+	end
+	
+end
+
+# Run a test script.
+if $0 == __FILE__
+	$stderr.print 'Press key to print demo.'; gets
+	code = File.read($0)[/module CodeRay.*/m]
+	print CodeRay.scan(code, :ruby).html
+end