coderay 0.9.8 → 1.0.0

data/{lib/README → README_INDEX.rdoc}

@@ -1,57 +1,45 @@
 = CodeRay
 
-[- Tired of blue'n'gray? Try the original version of this documentation on
-coderay.rubychan.de[http://coderay.rubychan.de/doc/] (use Ctrl+Click to open it in its own frame.) -]
+Tired of blue'n'gray? Try the original version of this documentation on
+coderay.rubychan.de[http://coderay.rubychan.de/doc/] :-)
 
 == About
+
 CodeRay is a Ruby library for syntax highlighting.
 
-Syntax highlighting means: You put your code in, and you get it back colored;
-Keywords, strings, floats, comments - all in different colors.
-And with line numbers.
+You put your code in, and you get it back colored; Keywords, strings,
+floats, comments - all in different colors. And with line numbers.
 
 *Syntax* *Highlighting*...
 * makes code easier to read and maintain
 * lets you detect syntax errors faster
 * helps you to understand the syntax of a language
 * looks nice
-* is what everybody should have on their website
+* is what everybody wants to have on their website
 * solves all your problems and makes the girls run after you
 
-Version: 0.9.8
-Author:: murphy (Kornelius Kalnbach)
-Contact:: murphy rubychan de
-Website:: coderay.rubychan.de[http://coderay.rubychan.de]
-License:: GNU LGPL; see LICENSE file in the main directory.
 
 == Installation
 
-You need RubyGems[http://rubyforge.org/frs/?group_id=126].
-
  % gem install coderay
 
 
 === Dependencies
 
-CodeRay needs Ruby 1.8.6 or later. It also runs with Ruby 1.9.1+ and JRuby 1.1+.
+CodeRay needs Ruby 1.8.7+ or 1.9.2+. It also runs on Rubinius and JRuby.
 
 
 == Example Usage
-(Forgive me, but this is not highlighted.)
 
  require 'coderay'
  
- tokens = CodeRay.scan "puts 'Hello, world!'", :ruby
- page = tokens.html :line_numbers => :inline, :wrap => :page
- puts page
+ html = CodeRay.scan("puts 'Hello, world!'", :ruby).div(:line_numbers => :table)
 
 
 == Documentation
 
 See CodeRay.
 
-Please report errors in this documentation to <murphy rubychan de>.
-
 
 == Credits
 
@@ -94,7 +82,6 @@ Please report errors in this documentation to <murphy rubychan de>.
 * Rob Aldred for the terminal encoder
 * Trans for pointing out $DEBUG dependencies
 * Flameeyes for finding that Term::ANSIColor was obsolete
-* Etienne Massip for reporting a serious bug in JavaScript scanner
 * matz and all Ruby gods and gurus
 * The inventors of: the computer, the internet, the true color display, HTML &
   CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime, 
@@ -124,6 +111,8 @@ Where would we be without all those people?
   less useless
 * Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
 * PLEAC[http://pleac.sourceforge.net/] code examples
+* Github
+* Travis CI (http://travis-ci.org/rubychan/github)
 
 === Free
 

data/Rakefile

@@ -1,8 +1,7 @@
-require 'rake/rdoctask'
+$:.unshift File.dirname(__FILE__) unless $:.include? '.'
 
 ROOT = '.'
 LIB_ROOT = File.join ROOT, 'lib'
-EXTRA_RDOC_FILES = %w(lib/README FOLDERS)
 
 task :default => :test
 
@@ -15,20 +14,21 @@ if File.directory? 'rake_tasks'
   
 else
   
-  # fallback tasks when rake_tasks folder is not present
+  # fallback tasks when rake_tasks folder is not present (eg. in the distribution package)
   desc 'Run CodeRay tests (basic)'
   task :test do
     ruby './test/functional/suite.rb'
     ruby './test/functional/for_redcloth.rb'
   end
   
+  gem 'rdoc' if defined? gem
+  require 'rdoc/task'
   desc 'Generate documentation for CodeRay'
   Rake::RDocTask.new :doc do |rd|
     rd.title = 'CodeRay Documentation'
-    rd.main = 'lib/README'
+    rd.main = 'README_INDEX.rdoc'
     rd.rdoc_files.add Dir['lib']
-    rd.rdoc_files.add 'lib/README'
-    rd.rdoc_files.add 'FOLDERS'
+    rd.rdoc_files.add rd.main
     rd.rdoc_dir = 'doc'
   end
   

data/bin/coderay

@@ -1,86 +1,215 @@
 #!/usr/bin/env ruby
-# CodeRay Executable
-#
-# Version: 0.2
-# Author: murphy
-
 require 'coderay'
 
-if ARGV.empty?
-  $stderr.puts <<-USAGE
-CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
+$options, args = ARGV.partition { |arg| arg[/^-[hv]$|--\w+/] }
+subcommand = args.first if /^\w/ === args.first
+subcommand = nil if subcommand && File.exist?(subcommand)
+args.delete subcommand
 
-Usage:
-  coderay file [-<format>]
-  coderay -<lang> [-<format>] [< file] [> output]
+def option? *options
+  !($options & options).empty?
+end
 
-Defaults:
-  lang:   based on file extension
-  format: ANSI colorized output for terminal, HTML page for files
+def tty?
+  $stdout.tty? || option?('--tty')
+end
 
-Examples:
-  coderay foo.rb                         # colorized output to terminal, based on file extension
-  coderay foo.rb -loc                    # print LOC count, based on file extension and format
-  coderay foo.rb > foo.html              # HTML page output to file, based on extension
-  coderay -ruby < foo.rb                 # colorized output to terminal, based on lang
-  coderay -ruby -loc < foo.rb            # print LOC count, based on lang
-  coderay -ruby -page foo.rb             # HTML page output to terminal, based on lang and format
-  coderay -ruby -page foo.rb > foo.html  # HTML page output to file, based on lang and format
+def version
+  puts <<-USAGE
+CodeRay #{CodeRay::VERSION}
   USAGE
 end
 
-first, second = ARGV
+def help
+  puts <<-HELP
+This is CodeRay #{CodeRay::VERSION}, a syntax highlighting tool for selected languages.
 
-def read
-  file = ARGV.grep(/^(?!-)/).last
-  if file
-    if File.exist?(file)
-      File.read file
-    else
-      $stderr.puts "No such file: #{file}"
-    end
-  else
-    $stdin.read
-  end
+usage:
+  coderay [-language] [input] [-format] [output]
+  
+defaults:
+  language   detect from input file name or shebang; fall back to plain text
+  input      STDIN
+  format     detect from output file name or use terminal; fall back to HTML
+  output     STDOUT
+
+common:
+  coderay file.rb                      # highlight file to terminal
+  coderay file.rb > file.html          # highlight file to HTML page
+  coderay file.rb -div > file.html     # highlight file to HTML snippet
+
+configure output:
+  coderay file.py output.json          # output tokens as JSON
+  coderay file.py -loc                 # count lines of code in Python file
+
+configure input:
+  coderay -python file                 # specify the input language
+  coderay -ruby                        # take input from STDIN
+
+more:
+  coderay stylesheet [style]           # print CSS stylesheet
+  HELP
 end
 
-if first
-  if first[/-(\w+)/] == first
-    lang = $1
-    input = read
-    tokens = :scan
-  else
-    file = first
-    unless File.exist? file
-      $stderr.puts "No such file: #{file}"
-      exit 2
+def commands
+  puts <<-COMMANDS
+  general:
+    highlight   code highlighting (default command, optional)
+    stylesheet  print the CSS stylesheet with the given name (aliases: style, css)
+  
+  about:
+    list [of]   list all available plugins (or just the scanners|encoders|styles|filetypes)
+    commands    print this list
+    help        show some help
+    version     print CodeRay version
+  COMMANDS
+end
+
+def print_list_of plugin_host
+  plugins = plugin_host.all_plugins.map do |plugin|
+    info = "  #{plugin.plugin_id}: #{plugin.title}"
+    
+    aliases = (plugin.aliases - [:default]).map { |key| "-#{key}" }.sort_by { |key| key.size }
+    if plugin.respond_to?(:file_extension) || !aliases.empty?
+      additional_info = []
+      additional_info << aliases.join(', ') unless aliases.empty?
+      info << " (#{additional_info.join('; ')})"
     end
-    tokens = CodeRay.scan_file file
+    
+    info << '  <-- default' if plugin.aliases.include? :default
+    
+    info
   end
-else
-  $stderr.puts 'No lang/file given.'
-  exit 1
+  puts plugins.sort
+end
+
+if option? '-v', '--version'
+  version
+end
+
+if option? '-h', '--help'
+  help
 end
 
-if second
-  if second[/-(\w+)/] == second
-    format = $1.to_sym
+case subcommand
+when 'highlight', nil
+  if ARGV.empty?
+    version
+    help
   else
-    raise 'invalid format (must be -xxx)'
+    signature = args.map { |arg| arg[/^-/] ? '-' : 'f' }.join
+    names     = args.map { |arg| arg.sub(/^-/, '') }
+    case signature
+    when /^$/
+      exit
+    when /^ff?$/
+      input_file, output_file, = *names
+    when /^f-f?$/
+      input_file, output_format, output_file, = *names
+    when /^-ff?$/
+      input_lang, input_file, output_file, = *names
+    when /^-f-f?$/
+      input_lang, input_file, output_format, output_file, = *names
+    when /^--?f?$/
+      input_lang, output_format, output_file, = *names
+    else
+      $stdout = $stderr
+      help
+      puts
+      puts "Unknown parameter order: #{args.join ' '}, expected: [-language] [input] [-format] [output]"
+      exit 1
+    end
+    
+    if input_file
+      input_lang ||= CodeRay::FileType.fetch input_file, :text, true
+    end
+    
+    if output_file
+      output_format ||= CodeRay::FileType[output_file]
+    else
+      output_format ||= :terminal
+    end
+    
+    output_format = :page if output_format.to_s == 'html'
+    
+    if input_file
+      input = File.read input_file
+    else
+      input = $stdin.read
+    end
+    
+    begin
+      file =
+        if output_file
+          File.open output_file, 'w'
+        else
+          $stdout.sync = true
+          $stdout
+        end
+      CodeRay.encode(input, input_lang, output_format, :out => file)
+      file.puts
+    rescue CodeRay::PluginHost::PluginNotFound => boom
+      $stdout = $stderr
+      if boom.message[/CodeRay::(\w+)s could not load plugin :?(.*?): /]
+        puts "I don't know the #$1 \"#$2\"."
+      else
+        puts boom.message
+      end
+      # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
+    rescue CodeRay::Scanners::Scanner::ScanError  # FIXME: rescue Errno::EPIPE
+      # this is sometimes raised by pagers; ignore [TODO: wtf?]
+    ensure
+      file.close if output_file
+    end
+  end
+when 'li', 'list'
+  arg = args.first && args.first.downcase
+  if [nil, 's', 'sc', 'scanner', 'scanners'].include? arg
+    puts 'input languages (Scanners):'
+    print_list_of CodeRay::Scanners
+  end
+  
+  if [nil, 'e', 'en', 'enc', 'encoder', 'encoders'].include? arg
+    puts 'output formats (Encoders):'
+    print_list_of CodeRay::Encoders
   end
+  
+  if [nil, 'st', 'style', 'styles'].include? arg
+    puts 'CSS themes for HTML output (Styles):'
+    print_list_of CodeRay::Styles
+  end
+  
+  if [nil, 'f', 'ft', 'file', 'filetype', 'filetypes'].include? arg
+    puts 'recognized file types:'
+    
+    filetypes = Hash.new { |h, k| h[k] = [] }
+    CodeRay::FileType::TypeFromExt.inject filetypes do |types, (ext, type)|
+      types[type.to_s] << ".#{ext}"
+      types
+    end
+    CodeRay::FileType::TypeFromName.inject filetypes do |types, (name, type)|
+      types[type.to_s] << name
+      types
+    end
+    
+    filetypes.sort.each do |type, exts|
+      puts "  #{type}: #{exts.sort_by { |ext| ext.size }.join(', ')}"
+    end
+  end
+when 'stylesheet', 'style', 'css'
+  puts CodeRay::Encoders[:html]::CSS.new(args.first).stylesheet
+when 'commands'
+  commands
+when 'help'
+  help
 else
-  if $stdout.tty?
-    format = :term
+  $stdout = $stderr
+  help
+  puts
+  if subcommand[/\A\w+\z/]
+    puts "Unknown command: #{subcommand}"
   else
-    $stderr.puts 'No format given; setting to default (HTML Page).'
-    format = :page
+    puts "File not found: #{subcommand}"
   end
+  exit 1
 end
-
-if tokens == :scan
-  output = CodeRay::Duo[lang => format].highlight input
-else
-  output = tokens.encode format
-end
-out = $stdout
-out.puts output

data/lib/coderay.rb

@@ -1,16 +1,21 @@
+# encoding: utf-8
+# Encoding.default_internal = 'UTF-8'
+
 # = CodeRay Library
 #
 # CodeRay is a Ruby library for syntax highlighting.
 #
-# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
-# fast and efficient.
+# I try to make CodeRay easy to use and intuitive, but at the same time fully
+# featured, complete, fast and efficient.
 # 
 # See README.
 # 
 # It consists mainly of
-# * the main engine: CodeRay (Scanners::Scanner, Tokens/TokenStream, Encoders::Encoder), PluginHost
+# * the main engine: CodeRay (Scanners::Scanner, Tokens, Encoders::Encoder)
+# * the plugin system: PluginHost, Plugin
 # * the scanners in CodeRay::Scanners
 # * the encoders in CodeRay::Encoders
+# * the styles in CodeRay::Styles
 # 
 # Here's a fancy graphic to light up this gray docu:
 # 
@@ -22,8 +27,8 @@
 #
 # == Usage
 #
-# Remember you need RubyGems to use CodeRay, unless you have it in your load path. Run Ruby with
-# -rubygems option if required.
+# Remember you need RubyGems to use CodeRay, unless you have it in your load
+# path. Run Ruby with -rubygems option if required.
 #
 # === Highlight Ruby code in a string as html
 # 
@@ -98,13 +103,6 @@
 # CodeRay.encode_tokens:: Encode the given tokens.
 # CodeRay.encode_file:: Scan a file, guess the language using FileType and encode it.
 #
-# == Streaming
-#
-# Streaming saves RAM by running Scanner and Encoder in some sort of
-# pipe mode; see TokenStream.
-#
-# CodeRay.scan_stream:: Scan in stream mode.
-#
 # == All-in-One Encoding
 #
 # CodeRay.encode:: Highlight a string with a given input and output format.
@@ -129,23 +127,30 @@ module CodeRay
   
   $CODERAY_DEBUG ||= false
   
-  # Version: Major.Minor.Teeny[.Revision]
-  # Major: 0 for pre-stable, 1 for stable
-  # Minor: feature milestone
-  # Teeny: development state, 0 for pre-release
-  # Revision: Subversion Revision number (generated on rake gem:make)
-  VERSION = '0.9.8'
-
-  require 'coderay/tokens'
-  require 'coderay/token_classes'
-  require 'coderay/scanner'
-  require 'coderay/encoder'
-  require 'coderay/duo'
-  require 'coderay/style'
-
-
+  require 'coderay/version'
+  
+  # helpers
+  autoload :FileType, 'coderay/helpers/file_type'
+  
+  # Tokens
+  autoload :Tokens, 'coderay/tokens'
+  autoload :TokensProxy, 'coderay/tokens_proxy'
+  autoload :TokenKinds, 'coderay/token_kinds'
+  
+  # Plugin system
+  autoload :PluginHost, 'coderay/helpers/plugin'
+  autoload :Plugin, 'coderay/helpers/plugin'
+  
+  # Plugins
+  autoload :Scanners, 'coderay/scanner'
+  autoload :Encoders, 'coderay/encoder'
+  autoload :Styles, 'coderay/style'
+  
+  # Convenience access and reusable Encoder/Scanner pair
+  autoload :Duo, 'coderay/duo'
+  
   class << self
-
+    
     # Scans the given +code+ (a String) with the Scanner for +lang+.
     #
     # This is a simple way to use CodeRay. Example:
@@ -154,15 +159,15 @@ module CodeRay
     #
     # See also demo/demo_simple.
     def scan code, lang, options = {}, &block
-      scanner = Scanners[lang].new code, options, &block
-      scanner.tokenize
+      # FIXME: return a proxy for direct-stream encoding
+      TokensProxy.new code, lang, options, block
     end
-
+    
     # Scans +filename+ (a path to a code file) with the Scanner for +lang+.
     #
     # If +lang+ is :auto or omitted, the CodeRay::FileType module is used to
     # determine it. If it cannot find out what type it is, it uses
-    # CodeRay::Scanners::Plaintext.
+    # CodeRay::Scanners::Text.
     #
     # Calls CodeRay.scan.
     #
@@ -170,56 +175,22 @@ module CodeRay
     #  require 'coderay'
     #  page = CodeRay.scan_file('some_c_code.c').html
     def scan_file filename, lang = :auto, options = {}, &block
-      file = IO.read filename
-      if lang == :auto
-        require 'coderay/helpers/file_type'
-        lang = FileType.fetch filename, :plaintext, true
-      end
-      scan file, lang, options = {}, &block
-    end
-
-    # Scan the +code+ (a string) with the scanner for +lang+.
-    #
-    # Calls scan.
-    #
-    # See CodeRay.scan.
-    def scan_stream code, lang, options = {}, &block
-      options[:stream] = true
+      lang = FileType.fetch filename, :text, true if lang == :auto
+      code = File.read filename
       scan code, lang, options, &block
     end
-
-    # Encode a string in Streaming mode.
-    #
-    # This starts scanning +code+ with the the Scanner for +lang+
-    # while encodes the output with the Encoder for +format+.
-    # +options+ will be passed to the Encoder.
-    #
-    # See CodeRay::Encoder.encode_stream
-    def encode_stream code, lang, format, options = {}
-      encoder(format, options).encode_stream code, lang, options
-    end
-
+    
     # Encode a string.
     #
     # This scans +code+ with the the Scanner for +lang+ and then
     # encodes it with the Encoder for +format+.
     # +options+ will be passed to the Encoder.
     #
-    # See CodeRay::Encoder.encode
+    # See CodeRay::Encoder.encode.
     def encode code, lang, format, options = {}
       encoder(format, options).encode code, lang, options
     end
-
-    # Highlight a string into a HTML <div>.
-    #
-    # CSS styles use classes, so you have to include a stylesheet
-    # in your output.
-    #
-    # See encode.
-    def highlight code, lang, options = { :css => :class }, format = :div
-      encode code, lang, format, options
-    end
-
+    
     # Encode pre-scanned Tokens.
     # Use this together with CodeRay.scan:
     #
@@ -232,7 +203,7 @@ module CodeRay
     def encode_tokens tokens, format, options = {}
       encoder(format, options).encode_tokens tokens, options
     end
-
+    
     # Encodes +filename+ (a path to a code file) with the Scanner for +lang+.
     #
     # See CodeRay.scan_file.
@@ -245,7 +216,17 @@ module CodeRay
       tokens = scan_file filename, :auto, get_scanner_options(options)
       encode_tokens tokens, format, options
     end
-
+    
+    # Highlight a string into a HTML <div>.
+    #
+    # CSS styles use classes, so you have to include a stylesheet
+    # in your output.
+    #
+    # See encode.
+    def highlight code, lang, options = { :css => :class }, format = :div
+      encode code, lang, format, options
+    end
+    
     # Highlight a file into a HTML <div>.
     #
     # CSS styles use classes, so you have to include a stylesheet
@@ -255,7 +236,7 @@ module CodeRay
     def highlight_file filename, options = { :css => :class }, format = :div
       encode_file filename, format, options
     end
-
+    
     # Finds the Encoder class for +format+ and creates an instance, passing
     # +options+ to it.
     #
@@ -273,15 +254,15 @@ module CodeRay
     def encoder format, options = {}
       Encoders[format].new options
     end
-
+    
     # Finds the Scanner class for +lang+ and creates an instance, passing
     # +options+ to it.
     #
     # See Scanner.new.
-    def scanner lang, options = {}
-      Scanners[lang].new '', options
+    def scanner lang, options = {}, &block
+      Scanners[lang].new '', options, &block
     end
-
+    
     # Extract the options for the scanner from the +options+ hash.
     #
     # Returns an empty Hash if <tt>:scanner_options</tt> is not set.
@@ -291,32 +272,7 @@ module CodeRay
     def get_scanner_options options
       options.fetch :scanner_options, {}
     end
-
+    
   end
-
-  # This Exception is raised when you try to stream with something that is not
-  # capable of streaming.
-  class NotStreamableError < Exception
-    def initialize obj
-      @obj = obj
-    end
-
-    def to_s
-      '%s is not Streamable!' % @obj.class
-    end
-  end
-
-  # A dummy module that is included by subclasses of CodeRay::Scanner an CodeRay::Encoder
-  # to show that they are able to handle streams.
-  module Streamable
-  end
-
-end
-
-# Run a test script.
-if $0 == __FILE__
-  $stderr.print 'Press key to print demo.'; gets
-  # Just use this file as an example of Ruby code.
-  code = File.read(__FILE__)[/module CodeRay.*/m]
-  print CodeRay.scan(code, :ruby).html
+  
 end