coderay 0.9.2 → 0.9.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,82 +1,100 @@
1
- #!/usr/bin/env ruby
2
- # CodeRay Executable
3
- #
4
- # Version: 0.1
5
- # Author: murphy
6
-
7
- def err msg
8
- $stderr.puts msg
9
- end
10
-
11
- begin
12
- require 'coderay'
13
-
14
- if ARGV.empty?
15
- puts <<-USAGE
16
- CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
17
- Usage:
18
- coderay -<lang> [-<format>] < file > output
19
- coderay file [-<format>]
20
- Example:
21
- coderay -ruby -statistic < foo.rb
22
- coderay codegen.c # generates codegen.c.html
23
- USAGE
24
- end
25
-
26
- first, second = ARGV
27
-
28
- if first
29
- if first[/-(\w+)/] == first
30
- lang = $1
31
- input = $stdin.read
32
- tokens = :scan
33
- elsif first == '-'
34
- lang = $1
35
- input = $stdin.read
36
- tokens = :scan
37
- else
38
- file = first
39
- tokens = CodeRay.scan_file file
40
- output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
41
- end
42
- else
43
- puts 'No lang/file given.'
44
- exit 1
45
- end
46
-
47
- if second
48
- if second[/-(\w+)/] == second
49
- format = $1
50
- else
51
- raise 'Invalid format (must be -xxx).'
52
- end
53
- else
54
- $stderr.puts 'No format given; setting to default (HTML Page)'
55
- format = :page
56
- end
57
-
58
- # TODO: allow streaming
59
- if tokens == :scan
60
- output = CodeRay::Duo[lang => format].highlight input #, :stream => true
61
- else
62
- output = tokens.encode format
63
- end
64
- out = $stdout
65
- if output_filename
66
- output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
67
- if File.exist? output_filename
68
- err 'File %s already exists.' % output_filename
69
- exit
70
- else
71
- out = File.open output_filename, 'w'
72
- end
73
- end
74
- out.print output
75
-
76
- rescue => boom
77
- err "Error: #{boom.message}\n"
78
- err boom.backtrace
79
- err '-' * 50
80
- err ARGV
81
- exit 1
82
- end
1
+ #!/usr/bin/env ruby
2
+ # CodeRay Executable
3
+ #
4
+ # Version: 0.2
5
+ # Author: murphy
6
+
7
+ def err msg
8
+ $stderr.puts msg
9
+ end
10
+
11
+ def read
12
+ if file = ARGV[2]
13
+ File.read file
14
+ else
15
+ $stdin.read
16
+ end
17
+ end
18
+
19
+ begin
20
+ require 'coderay'
21
+
22
+ if ARGV.empty?
23
+ puts <<-USAGE
24
+ CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
25
+
26
+ Usage:
27
+ coderay -<lang> [-<format>] < file > output
28
+ coderay file [-<format>]
29
+
30
+ Examples:
31
+ coderay -ruby -statistic < foo.rb
32
+ coderay -ruby < foo.rb # colorized output to terminal
33
+ coderay -ruby -page foo.rb # HTML page output to terminal
34
+ coderay -ruby -page foo.rb > foo.html # HTML page output to file
35
+ coderay codegen.c # generates codegen.c.html
36
+ USAGE
37
+ end
38
+
39
+ first, second = ARGV
40
+
41
+ if first
42
+ if first[/-(\w+)/] == first
43
+ lang = $1
44
+ input = read
45
+ tokens = :scan
46
+ elsif first == '-'
47
+ lang = $1
48
+ input = read
49
+ tokens = :scan
50
+ else
51
+ file = first
52
+ tokens = CodeRay.scan_file file
53
+ output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
54
+ end
55
+ else
56
+ puts 'No lang/file given.'
57
+ exit 1
58
+ end
59
+
60
+ if second
61
+ if second[/-(\w+)/] == second
62
+ format = $1
63
+ else
64
+ raise 'invalid format (must be -xxx)'
65
+ end
66
+ else
67
+ if $stdout.tty?
68
+ format = :term
69
+ else
70
+ $stderr.puts 'No format given; setting to default (HTML Page).'
71
+ format = :page
72
+ end
73
+ end
74
+
75
+ # TODO: allow streaming
76
+ if tokens == :scan
77
+ output = CodeRay::Duo[lang => format].highlight input #, :stream => true
78
+ else
79
+ output = tokens.encode format
80
+ end
81
+ out = $stdout
82
+ if output_filename
83
+ output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
84
+ if File.exist? output_filename
85
+ err 'File %s already exists.' % output_filename
86
+ exit
87
+ else
88
+ out = File.open output_filename, 'w'
89
+ puts "Writing to #{output_filename}..."
90
+ end
91
+ end
92
+ out.puts output
93
+
94
+ rescue => boom
95
+ err "Error: #{boom.message}\n"
96
+ err boom.backtrace
97
+ err '-' * 50
98
+ err ARGV
99
+ exit 1
100
+ end
data/lib/README CHANGED
@@ -18,7 +18,7 @@ And with line numbers.
18
18
  * is what everybody should have on their website
19
19
  * solves all your problems and makes the girls run after you
20
20
 
21
- Version: 0.9.2
21
+ Version: 0.9.3
22
22
  Author:: murphy (Kornelius Kalnbach)
23
23
  Contact:: murphy rubychan de
24
24
  Website:: coderay.rubychan.de[http://coderay.rubychan.de]
@@ -134,7 +134,7 @@ module CodeRay
134
134
  # Minor: feature milestone
135
135
  # Teeny: development state, 0 for pre-release
136
136
  # Revision: Subversion Revision number (generated on rake gem:make)
137
- VERSION = '0.9.2'
137
+ VERSION = '0.9.3'
138
138
 
139
139
  require 'coderay/tokens'
140
140
  require 'coderay/token_classes'
@@ -138,7 +138,7 @@ module Scanners
138
138
  elsif scan(/\\./m)
139
139
  kind = :content
140
140
  elsif scan(/ \\ | $ /x)
141
- tokens << [:close, :delimiter]
141
+ tokens << [:close, state]
142
142
  kind = :error
143
143
  state = :initial
144
144
  else
@@ -215,7 +215,7 @@ module Scanners
215
215
  end
216
216
 
217
217
  def xml_scanner
218
- @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
218
+ @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
219
219
  end
220
220
 
221
221
  end
@@ -13,9 +13,6 @@ module Scanners
13
13
  :error, :integer, :operator, :value,
14
14
  ]
15
15
 
16
- CONSTANTS = %w( true false null )
17
- IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
18
-
19
16
  ESCAPE = / [bfnrt\\"\/] /x
20
17
  UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
21
18
 
@@ -23,7 +20,6 @@ module Scanners
23
20
 
24
21
  state = :initial
25
22
  stack = []
26
- string_delimiter = nil
27
23
  key_expected = false
28
24
 
29
25
  until eos?
@@ -47,7 +43,7 @@ module Scanners
47
43
  when '}', ']' then stack.pop # no error recovery, but works for valid JSON
48
44
  end
49
45
  elsif match = scan(/ true | false | null /x)
50
- kind = IDENT_KIND[match]
46
+ kind = :value
51
47
  elsif match = scan(/-?(?:0|[1-9]\d*)/)
52
48
  kind = :integer
53
49
  if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
@@ -76,7 +72,7 @@ module Scanners
76
72
  elsif scan(/\\./m)
77
73
  kind = :content
78
74
  elsif scan(/ \\ | $ /x)
79
- tokens << [:close, :delimiter]
75
+ tokens << [:close, state]
80
76
  kind = :error
81
77
  state = :initial
82
78
  else
@@ -7,29 +7,30 @@ module CodeRay
7
7
  #
8
8
  # A token is not a special object, just a two-element Array
9
9
  # consisting of
10
+ # * the _token_ _text_ (the original source of the token in a String) or
11
+ # a _token_ _action_ (:open, :close, :begin_line, :end_line)
10
12
  # * the _token_ _kind_ (a Symbol representing the type of the token)
11
- # * the _token_ _text_ (the original source of the token in a String)
12
13
  #
13
14
  # A token looks like this:
14
15
  #
15
- # [:comment, '# It looks like this']
16
- # [:float, '3.1415926']
17
- # [:error, '$^']
16
+ # ['# It looks like this', :comment]
17
+ # ['3.1415926', :float]
18
+ # ['$^', :error]
18
19
  #
19
- # Some scanners also yield some kind of sub-tokens, represented by special
20
- # token texts, namely :open and :close .
20
+ # Some scanners also yield sub-tokens, represented by special
21
+ # token actions, namely :open and :close.
21
22
  #
22
23
  # The Ruby scanner, for example, splits "a string" into:
23
24
  #
24
25
  # [
25
26
  # [:open, :string],
26
- # [:delimiter, '"'],
27
- # [:content, 'a string'],
28
- # [:delimiter, '"'],
27
+ # ['"', :delimiter],
28
+ # ['a string', :content],
29
+ # ['"', :delimiter],
29
30
  # [:close, :string]
30
31
  # ]
31
32
  #
32
- # Tokens is also the interface between Scanners and Encoders:
33
+ # Tokens is the interface between Scanners and Encoders:
33
34
  # The input is split and saved into a Tokens object. The Encoder
34
35
  # then builds the output from this object.
35
36
  #
@@ -43,6 +44,9 @@ module CodeRay
43
44
  # Tokens gives you the power to handle pre-scanned code very easily:
44
45
  # You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
45
46
  # that you put in your DB.
47
+ #
48
+ # It also allows you to generate tokens directly (without using a scanner),
49
+ # to load them from a file, and still use any Encoder that CodeRay provides.
46
50
  #
47
51
  # Tokens' subclass TokenStream allows streaming to save memory.
48
52
  class Tokens < Array
@@ -239,9 +243,7 @@ module CodeRay
239
243
  size
240
244
  end
241
245
 
242
- # The total size of the tokens.
243
- # Should be equal to the input size before
244
- # scanning.
246
+ # Return all text tokens joined into a single string.
245
247
  def text
246
248
  map { |t, k| t if t.is_a? ::String }.join
247
249
  end
@@ -301,11 +303,11 @@ module CodeRay
301
303
  #
302
304
  # require 'coderay'
303
305
  #
304
- # token_stream = CodeRay::TokenStream.new do |kind, text|
306
+ # token_stream = CodeRay::TokenStream.new do |text, kind|
305
307
  # puts 'kind: %s, text size: %d.' % [kind, text.size]
306
308
  # end
307
309
  #
308
- # token_stream << [:regexp, '/\d+/']
310
+ # token_stream << ['/\d+/', :regexp]
309
311
  # #-> kind: rexpexp, text size: 5.
310
312
  #
311
313
  def initialize &block
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 9
8
- - 2
9
- version: 0.9.2
8
+ - 3
9
+ version: 0.9.3
10
10
  platform: ruby
11
11
  authors:
12
12
  - murphy
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-03-14 00:00:00 +01:00
17
+ date: 2010-04-19 00:00:00 +02:00
18
18
  default_executable:
19
19
  dependencies: []
20
20