coderay 1.0.9 → 1.1.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/Rakefile +2 -0
  3. data/bin/coderay +4 -4
  4. data/lib/coderay.rb +2 -3
  5. data/lib/coderay/encoders/debug.rb +5 -17
  6. data/lib/coderay/encoders/debug_lint.rb +62 -0
  7. data/lib/coderay/encoders/html.rb +84 -84
  8. data/lib/coderay/encoders/html/css.rb +7 -7
  9. data/lib/coderay/encoders/html/numbering.rb +24 -19
  10. data/lib/coderay/encoders/html/output.rb +1 -1
  11. data/lib/coderay/encoders/lint.rb +57 -0
  12. data/lib/coderay/encoders/statistic.rb +0 -1
  13. data/lib/coderay/encoders/terminal.rb +121 -105
  14. data/lib/coderay/helpers/file_type.rb +54 -47
  15. data/lib/coderay/helpers/plugin.rb +4 -13
  16. data/lib/coderay/scanner.rb +58 -26
  17. data/lib/coderay/scanners/c.rb +1 -1
  18. data/lib/coderay/scanners/cpp.rb +1 -1
  19. data/lib/coderay/scanners/css.rb +22 -25
  20. data/lib/coderay/scanners/diff.rb +53 -31
  21. data/lib/coderay/scanners/groovy.rb +17 -4
  22. data/lib/coderay/scanners/html.rb +38 -16
  23. data/lib/coderay/scanners/java.rb +1 -1
  24. data/lib/coderay/scanners/java_script.rb +30 -6
  25. data/lib/coderay/scanners/json.rb +15 -12
  26. data/lib/coderay/scanners/lua.rb +280 -0
  27. data/lib/coderay/scanners/php.rb +22 -4
  28. data/lib/coderay/scanners/python.rb +3 -3
  29. data/lib/coderay/scanners/raydebug.rb +8 -8
  30. data/lib/coderay/scanners/ruby.rb +2 -2
  31. data/lib/coderay/scanners/sass.rb +232 -0
  32. data/lib/coderay/scanners/sql.rb +7 -4
  33. data/lib/coderay/scanners/taskpaper.rb +36 -0
  34. data/lib/coderay/scanners/yaml.rb +2 -2
  35. data/lib/coderay/styles/alpha.rb +31 -21
  36. data/lib/coderay/token_kinds.rb +68 -71
  37. data/lib/coderay/tokens.rb +23 -77
  38. data/lib/coderay/version.rb +1 -1
  39. data/test/functional/examples.rb +3 -3
  40. data/test/functional/for_redcloth.rb +4 -10
  41. metadata +13 -14
  42. data/lib/coderay/helpers/gzip.rb +0 -41
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 88c4f8c3def05dd733e1e3b7707fb7e1f2d20d54
4
+ data.tar.gz: fe1abf58ddac324444cf7659d9d5d335bf470b46
5
+ SHA512:
6
+ metadata.gz: fad90cb9e8943b9cf55c8d54a5ab099d1012dcbdfa5208d58abb74aa6111d3d6d86cdb1f0f937b46533d813b59aa9ed1673942ab184618110349f38314117f2e
7
+ data.tar.gz: 1a2f782f48e427d181f557fded51236567200dd9b10a33a8668da98a5cccdb52872e4bbf0c68f2490a685b01430157bfb471ae892075cd25b56bb52e7fe7b56c
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ require 'bundler/gem_tasks'
2
+
1
3
  $:.unshift File.dirname(__FILE__) unless $:.include? '.'
2
4
 
3
5
  ROOT = '.'
data/bin/coderay CHANGED
@@ -125,7 +125,7 @@ when 'highlight', nil
125
125
  end
126
126
 
127
127
  if output_file
128
- output_format ||= CodeRay::FileType[output_file]
128
+ output_format ||= CodeRay::FileType[output_file] || :plain
129
129
  else
130
130
  output_format ||= :terminal
131
131
  end
@@ -143,7 +143,6 @@ when 'highlight', nil
143
143
  if output_file
144
144
  File.open output_file, 'w'
145
145
  else
146
- $stdout.sync = true
147
146
  $stdout
148
147
  end
149
148
  CodeRay.encode(input, input_lang, output_format, :out => file)
@@ -156,8 +155,9 @@ when 'highlight', nil
156
155
  puts boom.message
157
156
  end
158
157
  # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
159
- rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
160
- # this is sometimes raised by pagers; ignore [TODO: wtf?]
158
+ rescue CodeRay::Scanners::Scanner::ScanError
159
+ # this is sometimes raised by pagers; ignore
160
+ # FIXME: rescue Errno::EPIPE
161
161
  ensure
162
162
  file.close if output_file
163
163
  end
data/lib/coderay.rb CHANGED
@@ -127,14 +127,14 @@ module CodeRay
127
127
 
128
128
  $CODERAY_DEBUG ||= false
129
129
 
130
- CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay'
130
+ CODERAY_PATH = File.expand_path('../coderay', __FILE__)
131
131
 
132
132
  # Assuming the path is a subpath of lib/coderay/
133
133
  def self.coderay_path *path
134
134
  File.join CODERAY_PATH, *path
135
135
  end
136
136
 
137
- require coderay_path('version')
137
+ require 'coderay/version'
138
138
 
139
139
  # helpers
140
140
  autoload :FileType, coderay_path('helpers', 'file_type')
@@ -166,7 +166,6 @@ module CodeRay
166
166
  #
167
167
  # See also demo/demo_simple.
168
168
  def scan code, lang, options = {}, &block
169
- # FIXME: return a proxy for direct-stream encoding
170
169
  TokensProxy.new code, lang, options, block
171
170
  end
172
171
 
@@ -9,7 +9,6 @@ module Encoders
9
9
  #
10
10
  # You cannot fully restore the tokens information from the
11
11
  # output, because consecutive :space tokens are merged.
12
- # Use Tokens#dump for caching purposes.
13
12
  #
14
13
  # See also: Scanners::Debug
15
14
  class Debug < Encoder
@@ -18,37 +17,26 @@ module Encoders
18
17
 
19
18
  FILE_EXTENSION = 'raydebug'
20
19
 
21
- def initialize options = {}
22
- super
23
- @opened = []
24
- end
25
-
26
20
  def text_token text, kind
27
21
  if kind == :space
28
22
  @out << text
29
23
  else
30
- # TODO: Escape (
31
- text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
32
- @out << kind.to_s << '(' << text << ')'
24
+ text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
25
+ text = text.gsub(')', '\\\\)') if text.index(')')
26
+ @out << "#{kind}(#{text})"
33
27
  end
34
28
  end
35
29
 
36
30
  def begin_group kind
37
- @opened << kind
38
- @out << kind.to_s << '<'
31
+ @out << "#{kind}<"
39
32
  end
40
33
 
41
34
  def end_group kind
42
- if @opened.last != kind
43
- puts @out
44
- raise "we are inside #{@opened.inspect}, not #{kind}"
45
- end
46
- @opened.pop
47
35
  @out << '>'
48
36
  end
49
37
 
50
38
  def begin_line kind
51
- @out << kind.to_s << '['
39
+ @out << "#{kind}["
52
40
  end
53
41
 
54
42
  def end_line kind
@@ -0,0 +1,62 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ load :lint
5
+
6
+ # = Debug Lint Encoder
7
+ #
8
+ # Debug encoder with additional checks for:
9
+ #
10
+ # - empty tokens
11
+ # - incorrect nesting
12
+ #
13
+ # It will raise an InvalidTokenStream exception when any of the above occurs.
14
+ #
15
+ # See also: Encoders::Debug
16
+ class DebugLint < Debug
17
+
18
+ register_for :debug_lint
19
+
20
+ def text_token text, kind
21
+ raise Lint::EmptyToken, 'empty token' if text.empty?
22
+ super
23
+ end
24
+
25
+ def begin_group kind
26
+ @opened << kind
27
+ super
28
+ end
29
+
30
+ def end_group kind
31
+ raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
32
+ @opened.pop
33
+ super
34
+ end
35
+
36
+ def begin_line kind
37
+ @opened << kind
38
+ super
39
+ end
40
+
41
+ def end_line kind
42
+ raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
43
+ @opened.pop
44
+ super
45
+ end
46
+
47
+ protected
48
+
49
+ def setup options
50
+ super
51
+ @opened = []
52
+ end
53
+
54
+ def finish options
55
+ raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
56
+ super
57
+ end
58
+
59
+ end
60
+
61
+ end
62
+ end
@@ -126,22 +126,21 @@ module Encoders
126
126
 
127
127
  protected
128
128
 
129
- HTML_ESCAPE = { #:nodoc:
130
- '&' => '&amp;',
131
- '"' => '&quot;',
132
- '>' => '&gt;',
133
- '<' => '&lt;',
134
- }
129
+ def self.make_html_escape_hash
130
+ {
131
+ '&' => '&amp;',
132
+ '"' => '&quot;',
133
+ '>' => '&gt;',
134
+ '<' => '&lt;',
135
+ # "\t" => will be set to ' ' * options[:tab_width] during setup
136
+ }.tap do |hash|
137
+ # Escape ASCII control codes except \x9 == \t and \xA == \n.
138
+ (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
139
+ end
140
+ end
135
141
 
136
- # This was to prevent illegal HTML.
137
- # Strange chars should still be avoided in codes.
138
- evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
139
- evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
140
- #ansi_chars = Array(0x7f..0xff)
141
- #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
142
- # \x9 (\t) and \xA (\n) not included
143
- #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
144
- HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
142
+ HTML_ESCAPE = make_html_escape_hash
143
+ HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
145
144
 
146
145
  TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
147
146
  h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -172,66 +171,28 @@ module Encoders
172
171
  def setup options
173
172
  super
174
173
 
174
+ check_options! options
175
+
175
176
  if options[:wrap] || options[:line_numbers]
176
177
  @real_out = @out
177
178
  @out = ''
178
179
  end
179
180
 
180
- options[:break_lines] = true if options[:line_numbers] == :inline
181
-
182
181
  @break_lines = (options[:break_lines] == true)
183
182
 
184
- @HTML_ESCAPE = HTML_ESCAPE.dup
185
- @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
183
+ @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
186
184
 
187
185
  @opened = []
188
186
  @last_opened = nil
189
187
  @css = CSS.new options[:style]
190
188
 
191
- hint = options[:hint]
192
- if hint && ![:debug, :info, :info_long].include?(hint)
193
- raise ArgumentError, "Unknown value %p for :hint; \
194
- expected :info, :info_long, :debug, false, or nil." % hint
195
- end
196
-
197
- css_classes = TokenKinds
198
- case options[:css]
199
- when :class
200
- @span_for_kind = Hash.new do |h, k|
201
- if k.is_a? ::Symbol
202
- kind = k_dup = k
203
- else
204
- kind = k.first
205
- k_dup = k.dup
206
- end
207
- if kind != :space && (hint || css_class = css_classes[kind])
208
- title = HTML.token_path_to_hint hint, k if hint
209
- css_class ||= css_classes[kind]
210
- h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
211
- else
212
- h[k_dup] = nil
213
- end
214
- end
215
- when :style
216
- @span_for_kind = Hash.new do |h, k|
217
- kind = k.is_a?(Symbol) ? k : k.first
218
- h[k.is_a?(Symbol) ? k : k.dup] =
219
- if kind != :space && (hint || css_classes[kind])
220
- title = HTML.token_path_to_hint hint, k if hint
221
- style = @css.get_style Array(k).map { |c| css_classes[c] }
222
- "<span#{title}#{" style=\"#{style}\"" if style}>"
223
- end
224
- end
225
- else
226
- raise ArgumentError, "Unknown value %p for :css." % options[:css]
227
- end
189
+ @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
228
190
 
229
191
  @set_last_opened = options[:hint] || options[:css] == :style
230
192
  end
231
193
 
232
194
  def finish options
233
195
  unless @opened.empty?
234
- warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
235
196
  @out << '</span>' while @opened.pop
236
197
  @last_opened = nil
237
198
  end
@@ -255,20 +216,10 @@ module Encoders
255
216
  public
256
217
 
257
218
  def text_token text, kind
258
- if text =~ /#{HTML_ESCAPE_PATTERN}/o
259
- text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
260
- end
219
+ style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
261
220
 
262
- style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
263
-
264
- if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
265
- close = '</span>' * c
266
- reopen = ''
267
- @opened.each_with_index do |k, index|
268
- reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
269
- end
270
- text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
271
- end
221
+ text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
222
+ text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
272
223
 
273
224
  if style
274
225
  @out << style << text << '</span>'
@@ -279,25 +230,19 @@ module Encoders
279
230
 
280
231
  # token groups, eg. strings
281
232
  def begin_group kind
282
- @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
233
+ @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
283
234
  @opened << kind
284
235
  @last_opened = kind if @set_last_opened
285
236
  end
286
237
 
287
238
  def end_group kind
288
- if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
289
- warn 'Malformed token stream: Trying to close a token (%p) ' \
290
- 'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
291
- end
292
- if @opened.pop
293
- @out << '</span>'
294
- @last_opened = @opened.last if @last_opened
295
- end
239
+ check_group_nesting 'token group', kind if $CODERAY_DEBUG
240
+ close_span
296
241
  end
297
242
 
298
243
  # whole lines to be highlighted, eg. a deleted line in a diff
299
244
  def begin_line kind
300
- if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
245
+ if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
301
246
  if style['class="']
302
247
  @out << style.sub('class="', 'class="line ')
303
248
  else
@@ -311,16 +256,71 @@ module Encoders
311
256
  end
312
257
 
313
258
  def end_line kind
314
- if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
315
- warn 'Malformed token stream: Trying to close a line (%p) ' \
316
- 'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
259
+ check_group_nesting 'line', kind if $CODERAY_DEBUG
260
+ close_span
261
+ end
262
+
263
+ protected
264
+
265
+ def check_options! options
266
+ unless [false, nil, :debug, :info, :info_long].include? options[:hint]
267
+ raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
268
+ end
269
+
270
+ unless [:class, :style].include? options[:css]
271
+ raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
272
+ end
273
+
274
+ options[:break_lines] = true if options[:line_numbers] == :inline
275
+ end
276
+
277
+ def css_class_for_kinds kinds
278
+ TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
279
+ end
280
+
281
+ def style_for_kinds kinds
282
+ css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
283
+ @css.get_style_for_css_classes css_classes
284
+ end
285
+
286
+ def make_span_for_kinds method, hint
287
+ Hash.new do |h, kinds|
288
+ h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
289
+ css_class = css_class_for_kinds(kinds)
290
+ title = HTML.token_path_to_hint hint, kinds if hint
291
+
292
+ if css_class || title
293
+ if method == :style
294
+ style = style_for_kinds(kinds)
295
+ "<span#{title}#{" style=\"#{style}\"" if style}>"
296
+ else
297
+ "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
298
+ end
299
+ end
300
+ end
301
+ end
302
+ end
303
+
304
+ def check_group_nesting name, kind
305
+ if @opened.empty? || @opened.last != kind
306
+ warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
317
307
  end
308
+ end
309
+
310
+ def break_lines text, style
311
+ reopen = ''
312
+ @opened.each_with_index do |k, index|
313
+ reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
314
+ end
315
+ text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
316
+ end
317
+
318
+ def close_span
318
319
  if @opened.pop
319
320
  @out << '</span>'
320
321
  @last_opened = @opened.last if @last_opened
321
322
  end
322
323
  end
323
-
324
324
  end
325
325
 
326
326
  end
@@ -11,7 +11,7 @@ module Encoders
11
11
  end
12
12
 
13
13
  def initialize style = :default
14
- @classes = Hash.new
14
+ @styles = Hash.new
15
15
  style = CSS.load_stylesheet style
16
16
  @stylesheet = [
17
17
  style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ module Encoders
20
20
  parse style::TOKEN_COLORS
21
21
  end
22
22
 
23
- def get_style styles
24
- cl = @classes[styles.first]
23
+ def get_style_for_css_classes css_classes
24
+ cl = @styles[css_classes.first]
25
25
  return '' unless cl
26
26
  style = ''
27
- 1.upto styles.size do |offset|
28
- break if style = cl[styles[offset .. -1]]
27
+ 1.upto css_classes.size do |offset|
28
+ break if style = cl[css_classes[offset .. -1]]
29
29
  end
30
30
  # warn 'Style not found: %p' % [styles] if style.empty?
31
31
  return style
@@ -52,8 +52,8 @@ module Encoders
52
52
  for selector in selectors.split(',')
53
53
  classes = selector.scan(/[-\w]+/)
54
54
  cl = classes.pop
55
- @classes[cl] ||= Hash.new
56
- @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
55
+ @styles[cl] ||= Hash.new
56
+ @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
57
57
  end
58
58
  end
59
59
  end