coderay 0.9.2 → 0.9.3

data/bin/coderay

@@ -1,82 +1,100 @@
-#!/usr/bin/env ruby
-# CodeRay Executable
-#
-# Version: 0.1
-# Author: murphy
-
-def err msg
-  $stderr.puts msg
-end
-
-begin
-  require 'coderay'
-
-  if ARGV.empty?
-    puts <<-USAGE
-CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
-Usage:
-  coderay -<lang> [-<format>] < file > output
-  coderay file [-<format>]
-Example:
-  coderay -ruby -statistic < foo.rb
-  coderay codegen.c  # generates codegen.c.html
-    USAGE
-  end
-
-  first, second = ARGV
-
-  if first
-    if first[/-(\w+)/] == first
-      lang = $1
-      input = $stdin.read
-      tokens = :scan
-    elsif first == '-'
-      lang = $1
-      input = $stdin.read
-      tokens = :scan
-    else
-      file = first
-      tokens = CodeRay.scan_file file
-      output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
-    end
-  else
-    puts 'No lang/file given.'
-    exit 1
-  end
-
-  if second
-    if second[/-(\w+)/] == second
-      format = $1
-    else
-      raise 'Invalid format (must be -xxx).'
-    end
-  else
-    $stderr.puts 'No format given; setting to default (HTML Page)'
-    format = :page
-  end
-  
-  # TODO: allow streaming
-  if tokens == :scan
-    output = CodeRay::Duo[lang => format].highlight input  #, :stream => true
-  else
-    output = tokens.encode format
-  end
-  out = $stdout
-  if output_filename
-    output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
-    if File.exist? output_filename
-      err 'File %s already exists.' % output_filename
-      exit
-    else
-      out = File.open output_filename, 'w'
-    end
-  end
-  out.print output
-
-rescue => boom
-  err "Error: #{boom.message}\n"
-  err boom.backtrace
-  err '-' * 50
-  err ARGV
-  exit 1
-end
+#!/usr/bin/env ruby
+# CodeRay Executable
+#
+# Version: 0.2
+# Author: murphy
+
+def err msg
+  $stderr.puts msg
+end
+
+def read
+  if file = ARGV[2]
+    File.read file
+  else
+    $stdin.read
+  end
+end
+
+begin
+  require 'coderay'
+
+  if ARGV.empty?
+    puts <<-USAGE
+CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
+
+Usage:
+  coderay -<lang> [-<format>] < file > output
+  coderay file [-<format>]
+
+Examples:
+  coderay -ruby -statistic < foo.rb
+  coderay -ruby < foo.rb  # colorized output to terminal
+  coderay -ruby -page foo.rb  # HTML page output to terminal
+  coderay -ruby -page foo.rb > foo.html  # HTML page output to file
+  coderay codegen.c  # generates codegen.c.html
+    USAGE
+  end
+
+  first, second = ARGV
+
+  if first
+    if first[/-(\w+)/] == first
+      lang = $1
+      input = read
+      tokens = :scan
+    elsif first == '-'
+      lang = $1
+      input = read
+      tokens = :scan
+    else
+      file = first
+      tokens = CodeRay.scan_file file
+      output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
+    end
+  else
+    puts 'No lang/file given.'
+    exit 1
+  end
+
+  if second
+    if second[/-(\w+)/] == second
+      format = $1
+    else
+      raise 'invalid format (must be -xxx)'
+    end
+  else
+    if $stdout.tty?
+      format = :term
+    else
+      $stderr.puts 'No format given; setting to default (HTML Page).'
+      format = :page
+    end
+  end
+  
+  # TODO: allow streaming
+  if tokens == :scan
+    output = CodeRay::Duo[lang => format].highlight input  #, :stream => true
+  else
+    output = tokens.encode format
+  end
+  out = $stdout
+  if output_filename
+    output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
+    if File.exist? output_filename
+      err 'File %s already exists.' % output_filename
+      exit
+    else
+      out = File.open output_filename, 'w'
+      puts "Writing to #{output_filename}..."
+    end
+  end
+  out.puts output
+
+rescue => boom
+  err "Error: #{boom.message}\n"
+  err boom.backtrace
+  err '-' * 50
+  err ARGV
+  exit 1
+end

data/lib/README

@@ -18,7 +18,7 @@ And with line numbers.
 * is what everybody should have on their website
 * solves all your problems and makes the girls run after you
 
-Version: 0.9.2
+Version: 0.9.3
 Author:: murphy (Kornelius Kalnbach)
 Contact:: murphy rubychan de
 Website:: coderay.rubychan.de[http://coderay.rubychan.de]

data/lib/coderay.rb

@@ -134,7 +134,7 @@ module CodeRay
   # Minor: feature milestone
   # Teeny: development state, 0 for pre-release
   # Revision: Subversion Revision number (generated on rake gem:make)
-  VERSION = '0.9.2'
+  VERSION = '0.9.3'
 
   require 'coderay/tokens'
   require 'coderay/token_classes'

data/lib/coderay/scanners/java.rb

@@ -138,7 +138,7 @@ module Scanners
           elsif scan(/\\./m)
             kind = :content
           elsif scan(/ \\ | $ /x)
-            tokens << [:close, :delimiter]
+            tokens << [:close, state]
             kind = :error
             state = :initial
           else

data/lib/coderay/scanners/java_script.rb

@@ -215,7 +215,7 @@ module Scanners
     end
 
     def xml_scanner
-      @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
+      @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
     end
 
   end

data/lib/coderay/scanners/json.rb

@@ -13,9 +13,6 @@ module Scanners
       :error, :integer, :operator, :value,
     ]
     
-    CONSTANTS = %w( true false null )
-    IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
-    
     ESCAPE = / [bfnrt\\"\/] /x
     UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} /x
     
@@ -23,7 +20,6 @@ module Scanners
       
       state = :initial
       stack = []
-      string_delimiter = nil
       key_expected = false
       
       until eos?
@@ -47,7 +43,7 @@ module Scanners
             when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
             end
           elsif match = scan(/ true | false | null /x)
-            kind = IDENT_KIND[match]
+            kind = :value
           elsif match = scan(/-?(?:0|[1-9]\d*)/)
             kind = :integer
             if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
@@ -76,7 +72,7 @@ module Scanners
           elsif scan(/\\./m)
             kind = :content
           elsif scan(/ \\ | $ /x)
-            tokens << [:close, :delimiter]
+            tokens << [:close, state]
             kind = :error
             state = :initial
           else

data/lib/coderay/tokens.rb

@@ -7,29 +7,30 @@ module CodeRay
   #
   # A token is not a special object, just a two-element Array
   # consisting of
+  # * the _token_ _text_ (the original source of the token in a String) or
+  #   a _token_ _action_ (:open, :close, :begin_line, :end_line)
   # * the _token_ _kind_ (a Symbol representing the type of the token)
-  # * the _token_ _text_ (the original source of the token in a String)
   #
   # A token looks like this:
   #
-  #   [:comment, '# It looks like this']
-  #   [:float, '3.1415926']
-  #   [:error, '$^']
+  #   ['# It looks like this', :comment]
+  #   ['3.1415926', :float]
+  #   ['$^', :error]
   #
-  # Some scanners also yield some kind of sub-tokens, represented by special
-  # token texts, namely :open and :close .
+  # Some scanners also yield sub-tokens, represented by special
+  # token actions, namely :open and :close.
   #
   # The Ruby scanner, for example, splits "a string" into:
   #
   #  [
   #   [:open, :string],
-  #   [:delimiter, '"'],
-  #   [:content, 'a string'],
-  #   [:delimiter, '"'],
+  #   ['"', :delimiter],
+  #   ['a string', :content],
+  #   ['"', :delimiter],
   #   [:close, :string]
   #  ]
   #
-  # Tokens is also the interface between Scanners and Encoders:
+  # Tokens is the interface between Scanners and Encoders:
   # The input is split and saved into a Tokens object. The Encoder
   # then builds the output from this object.
   #
@@ -43,6 +44,9 @@ module CodeRay
   # Tokens gives you the power to handle pre-scanned code very easily:
   # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
   # that you put in your DB.
+  # 
+  # It also allows you to generate tokens directly (without using a scanner),
+  # to load them from a file, and still use any Encoder that CodeRay provides.
   #
   # Tokens' subclass TokenStream allows streaming to save memory.
   class Tokens < Array
@@ -239,9 +243,7 @@ module CodeRay
       size
     end
 
-    # The total size of the tokens.
-    # Should be equal to the input size before
-    # scanning.
+    # Return all text tokens joined into a single string.
     def text
       map { |t, k| t if t.is_a? ::String }.join
     end
@@ -301,11 +303,11 @@ module CodeRay
     #
     #   require 'coderay'
     #   
-    #   token_stream = CodeRay::TokenStream.new do |kind, text|
+    #   token_stream = CodeRay::TokenStream.new do |text, kind|
     #     puts 'kind: %s, text size: %d.' % [kind, text.size]
     #   end
     #   
-    #   token_stream << [:regexp, '/\d+/']
+    #   token_stream << ['/\d+/', :regexp]
     #   #-> kind: rexpexp, text size: 5.
     #
     def initialize &block

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 9
-  - 2
-  version: 0.9.2
+  - 3
+  version: 0.9.3
 platform: ruby
 authors: 
 - murphy
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-14 00:00:00 +01:00
+date: 2010-04-19 00:00:00 +02:00
 default_executable: 
 dependencies: []