coderay 1.0.9 → 1.1.0.rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/Rakefile +2 -0
  3. data/bin/coderay +4 -4
  4. data/lib/coderay.rb +2 -3
  5. data/lib/coderay/encoders/debug.rb +5 -17
  6. data/lib/coderay/encoders/debug_lint.rb +62 -0
  7. data/lib/coderay/encoders/html.rb +84 -84
  8. data/lib/coderay/encoders/html/css.rb +7 -7
  9. data/lib/coderay/encoders/html/numbering.rb +24 -19
  10. data/lib/coderay/encoders/html/output.rb +1 -1
  11. data/lib/coderay/encoders/lint.rb +57 -0
  12. data/lib/coderay/encoders/statistic.rb +0 -1
  13. data/lib/coderay/encoders/terminal.rb +121 -105
  14. data/lib/coderay/helpers/file_type.rb +54 -47
  15. data/lib/coderay/helpers/plugin.rb +4 -13
  16. data/lib/coderay/scanner.rb +58 -26
  17. data/lib/coderay/scanners/c.rb +1 -1
  18. data/lib/coderay/scanners/cpp.rb +1 -1
  19. data/lib/coderay/scanners/css.rb +22 -25
  20. data/lib/coderay/scanners/diff.rb +53 -31
  21. data/lib/coderay/scanners/groovy.rb +17 -4
  22. data/lib/coderay/scanners/html.rb +38 -16
  23. data/lib/coderay/scanners/java.rb +1 -1
  24. data/lib/coderay/scanners/java_script.rb +30 -6
  25. data/lib/coderay/scanners/json.rb +15 -12
  26. data/lib/coderay/scanners/lua.rb +280 -0
  27. data/lib/coderay/scanners/php.rb +22 -4
  28. data/lib/coderay/scanners/python.rb +3 -3
  29. data/lib/coderay/scanners/raydebug.rb +8 -8
  30. data/lib/coderay/scanners/ruby.rb +2 -2
  31. data/lib/coderay/scanners/sass.rb +232 -0
  32. data/lib/coderay/scanners/sql.rb +7 -4
  33. data/lib/coderay/scanners/taskpaper.rb +36 -0
  34. data/lib/coderay/scanners/yaml.rb +2 -2
  35. data/lib/coderay/styles/alpha.rb +31 -21
  36. data/lib/coderay/token_kinds.rb +68 -71
  37. data/lib/coderay/tokens.rb +23 -77
  38. data/lib/coderay/version.rb +1 -1
  39. data/test/functional/examples.rb +3 -3
  40. data/test/functional/for_redcloth.rb +4 -10
  41. metadata +13 -14
  42. data/lib/coderay/helpers/gzip.rb +0 -41
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 88c4f8c3def05dd733e1e3b7707fb7e1f2d20d54
4
+ data.tar.gz: fe1abf58ddac324444cf7659d9d5d335bf470b46
5
+ SHA512:
6
+ metadata.gz: fad90cb9e8943b9cf55c8d54a5ab099d1012dcbdfa5208d58abb74aa6111d3d6d86cdb1f0f937b46533d813b59aa9ed1673942ab184618110349f38314117f2e
7
+ data.tar.gz: 1a2f782f48e427d181f557fded51236567200dd9b10a33a8668da98a5cccdb52872e4bbf0c68f2490a685b01430157bfb471ae892075cd25b56bb52e7fe7b56c
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ require 'bundler/gem_tasks'
2
+
1
3
  $:.unshift File.dirname(__FILE__) unless $:.include? '.'
2
4
 
3
5
  ROOT = '.'
data/bin/coderay CHANGED
@@ -125,7 +125,7 @@ when 'highlight', nil
125
125
  end
126
126
 
127
127
  if output_file
128
- output_format ||= CodeRay::FileType[output_file]
128
+ output_format ||= CodeRay::FileType[output_file] || :plain
129
129
  else
130
130
  output_format ||= :terminal
131
131
  end
@@ -143,7 +143,6 @@ when 'highlight', nil
143
143
  if output_file
144
144
  File.open output_file, 'w'
145
145
  else
146
- $stdout.sync = true
147
146
  $stdout
148
147
  end
149
148
  CodeRay.encode(input, input_lang, output_format, :out => file)
@@ -156,8 +155,9 @@ when 'highlight', nil
156
155
  puts boom.message
157
156
  end
158
157
  # puts "I don't know this plugin: #{boom.message[/Could not load plugin (.*?): /, 1]}."
159
- rescue CodeRay::Scanners::Scanner::ScanError # FIXME: rescue Errno::EPIPE
160
- # this is sometimes raised by pagers; ignore [TODO: wtf?]
158
+ rescue CodeRay::Scanners::Scanner::ScanError
159
+ # this is sometimes raised by pagers; ignore
160
+ # FIXME: rescue Errno::EPIPE
161
161
  ensure
162
162
  file.close if output_file
163
163
  end
data/lib/coderay.rb CHANGED
@@ -127,14 +127,14 @@ module CodeRay
127
127
 
128
128
  $CODERAY_DEBUG ||= false
129
129
 
130
- CODERAY_PATH = File.join File.dirname(__FILE__), 'coderay'
130
+ CODERAY_PATH = File.expand_path('../coderay', __FILE__)
131
131
 
132
132
  # Assuming the path is a subpath of lib/coderay/
133
133
  def self.coderay_path *path
134
134
  File.join CODERAY_PATH, *path
135
135
  end
136
136
 
137
- require coderay_path('version')
137
+ require 'coderay/version'
138
138
 
139
139
  # helpers
140
140
  autoload :FileType, coderay_path('helpers', 'file_type')
@@ -166,7 +166,6 @@ module CodeRay
166
166
  #
167
167
  # See also demo/demo_simple.
168
168
  def scan code, lang, options = {}, &block
169
- # FIXME: return a proxy for direct-stream encoding
170
169
  TokensProxy.new code, lang, options, block
171
170
  end
172
171
 
@@ -9,7 +9,6 @@ module Encoders
9
9
  #
10
10
  # You cannot fully restore the tokens information from the
11
11
  # output, because consecutive :space tokens are merged.
12
- # Use Tokens#dump for caching purposes.
13
12
  #
14
13
  # See also: Scanners::Debug
15
14
  class Debug < Encoder
@@ -18,37 +17,26 @@ module Encoders
18
17
 
19
18
  FILE_EXTENSION = 'raydebug'
20
19
 
21
- def initialize options = {}
22
- super
23
- @opened = []
24
- end
25
-
26
20
  def text_token text, kind
27
21
  if kind == :space
28
22
  @out << text
29
23
  else
30
- # TODO: Escape (
31
- text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
32
- @out << kind.to_s << '(' << text << ')'
24
+ text = text.gsub('\\', '\\\\\\\\') if text.index('\\')
25
+ text = text.gsub(')', '\\\\)') if text.index(')')
26
+ @out << "#{kind}(#{text})"
33
27
  end
34
28
  end
35
29
 
36
30
  def begin_group kind
37
- @opened << kind
38
- @out << kind.to_s << '<'
31
+ @out << "#{kind}<"
39
32
  end
40
33
 
41
34
  def end_group kind
42
- if @opened.last != kind
43
- puts @out
44
- raise "we are inside #{@opened.inspect}, not #{kind}"
45
- end
46
- @opened.pop
47
35
  @out << '>'
48
36
  end
49
37
 
50
38
  def begin_line kind
51
- @out << kind.to_s << '['
39
+ @out << "#{kind}["
52
40
  end
53
41
 
54
42
  def end_line kind
@@ -0,0 +1,62 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ load :lint
5
+
6
+ # = Debug Lint Encoder
7
+ #
8
+ # Debug encoder with additional checks for:
9
+ #
10
+ # - empty tokens
11
+ # - incorrect nesting
12
+ #
13
+ # It will raise an InvalidTokenStream exception when any of the above occurs.
14
+ #
15
+ # See also: Encoders::Debug
16
+ class DebugLint < Debug
17
+
18
+ register_for :debug_lint
19
+
20
+ def text_token text, kind
21
+ raise Lint::EmptyToken, 'empty token' if text.empty?
22
+ super
23
+ end
24
+
25
+ def begin_group kind
26
+ @opened << kind
27
+ super
28
+ end
29
+
30
+ def end_group kind
31
+ raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
32
+ @opened.pop
33
+ super
34
+ end
35
+
36
+ def begin_line kind
37
+ @opened << kind
38
+ super
39
+ end
40
+
41
+ def end_line kind
42
+ raise Lint::IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
43
+ @opened.pop
44
+ super
45
+ end
46
+
47
+ protected
48
+
49
+ def setup options
50
+ super
51
+ @opened = []
52
+ end
53
+
54
+ def finish options
55
+ raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
56
+ super
57
+ end
58
+
59
+ end
60
+
61
+ end
62
+ end
@@ -126,22 +126,21 @@ module Encoders
126
126
 
127
127
  protected
128
128
 
129
- HTML_ESCAPE = { #:nodoc:
130
- '&' => '&amp;',
131
- '"' => '&quot;',
132
- '>' => '&gt;',
133
- '<' => '&lt;',
134
- }
129
+ def self.make_html_escape_hash
130
+ {
131
+ '&' => '&amp;',
132
+ '"' => '&quot;',
133
+ '>' => '&gt;',
134
+ '<' => '&lt;',
135
+ # "\t" => will be set to ' ' * options[:tab_width] during setup
136
+ }.tap do |hash|
137
+ # Escape ASCII control codes except \x9 == \t and \xA == \n.
138
+ (Array(0x00..0x8) + Array(0xB..0x1F)).each { |invalid| hash[invalid.chr] = ' ' }
139
+ end
140
+ end
135
141
 
136
- # This was to prevent illegal HTML.
137
- # Strange chars should still be avoided in codes.
138
- evil_chars = Array(0x00...0x20) - [?\n, ?\t, ?\s]
139
- evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
140
- #ansi_chars = Array(0x7f..0xff)
141
- #ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
142
- # \x9 (\t) and \xA (\n) not included
143
- #HTML_ESCAPE_PATTERN = /[\t&"><\0-\x8\xB-\x1f\x7f-\xff]/
144
- HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1f]/
142
+ HTML_ESCAPE = make_html_escape_hash
143
+ HTML_ESCAPE_PATTERN = /[\t"&><\0-\x8\xB-\x1F]/
145
144
 
146
145
  TOKEN_KIND_TO_INFO = Hash.new do |h, kind|
147
146
  h[kind] = kind.to_s.gsub(/_/, ' ').gsub(/\b\w/) { $&.capitalize }
@@ -172,66 +171,28 @@ module Encoders
172
171
  def setup options
173
172
  super
174
173
 
174
+ check_options! options
175
+
175
176
  if options[:wrap] || options[:line_numbers]
176
177
  @real_out = @out
177
178
  @out = ''
178
179
  end
179
180
 
180
- options[:break_lines] = true if options[:line_numbers] == :inline
181
-
182
181
  @break_lines = (options[:break_lines] == true)
183
182
 
184
- @HTML_ESCAPE = HTML_ESCAPE.dup
185
- @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
183
+ @HTML_ESCAPE = HTML_ESCAPE.merge("\t" => ' ' * options[:tab_width])
186
184
 
187
185
  @opened = []
188
186
  @last_opened = nil
189
187
  @css = CSS.new options[:style]
190
188
 
191
- hint = options[:hint]
192
- if hint && ![:debug, :info, :info_long].include?(hint)
193
- raise ArgumentError, "Unknown value %p for :hint; \
194
- expected :info, :info_long, :debug, false, or nil." % hint
195
- end
196
-
197
- css_classes = TokenKinds
198
- case options[:css]
199
- when :class
200
- @span_for_kind = Hash.new do |h, k|
201
- if k.is_a? ::Symbol
202
- kind = k_dup = k
203
- else
204
- kind = k.first
205
- k_dup = k.dup
206
- end
207
- if kind != :space && (hint || css_class = css_classes[kind])
208
- title = HTML.token_path_to_hint hint, k if hint
209
- css_class ||= css_classes[kind]
210
- h[k_dup] = "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
211
- else
212
- h[k_dup] = nil
213
- end
214
- end
215
- when :style
216
- @span_for_kind = Hash.new do |h, k|
217
- kind = k.is_a?(Symbol) ? k : k.first
218
- h[k.is_a?(Symbol) ? k : k.dup] =
219
- if kind != :space && (hint || css_classes[kind])
220
- title = HTML.token_path_to_hint hint, k if hint
221
- style = @css.get_style Array(k).map { |c| css_classes[c] }
222
- "<span#{title}#{" style=\"#{style}\"" if style}>"
223
- end
224
- end
225
- else
226
- raise ArgumentError, "Unknown value %p for :css." % options[:css]
227
- end
189
+ @span_for_kinds = make_span_for_kinds(options[:css], options[:hint])
228
190
 
229
191
  @set_last_opened = options[:hint] || options[:css] == :style
230
192
  end
231
193
 
232
194
  def finish options
233
195
  unless @opened.empty?
234
- warn '%d tokens still open: %p' % [@opened.size, @opened] if $CODERAY_DEBUG
235
196
  @out << '</span>' while @opened.pop
236
197
  @last_opened = nil
237
198
  end
@@ -255,20 +216,10 @@ module Encoders
255
216
  public
256
217
 
257
218
  def text_token text, kind
258
- if text =~ /#{HTML_ESCAPE_PATTERN}/o
259
- text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
260
- end
219
+ style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
261
220
 
262
- style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
263
-
264
- if @break_lines && (i = text.index("\n")) && (c = @opened.size + (style ? 1 : 0)) > 0
265
- close = '</span>' * c
266
- reopen = ''
267
- @opened.each_with_index do |k, index|
268
- reopen << (@span_for_kind[index > 0 ? [k, *@opened[0 ... index ]] : k] || '<span>')
269
- end
270
- text[i .. -1] = text[i .. -1].gsub("\n", "#{close}\n#{reopen}#{style}")
271
- end
221
+ text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } if text =~ /#{HTML_ESCAPE_PATTERN}/o
222
+ text = break_lines(text, style) if @break_lines && (style || @opened.size > 0) && text.index("\n")
272
223
 
273
224
  if style
274
225
  @out << style << text << '</span>'
@@ -279,25 +230,19 @@ module Encoders
279
230
 
280
231
  # token groups, eg. strings
281
232
  def begin_group kind
282
- @out << (@span_for_kind[@last_opened ? [kind, *@opened] : kind] || '<span>')
233
+ @out << (@span_for_kinds[@last_opened ? [kind, *@opened] : kind] || '<span>')
283
234
  @opened << kind
284
235
  @last_opened = kind if @set_last_opened
285
236
  end
286
237
 
287
238
  def end_group kind
288
- if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
289
- warn 'Malformed token stream: Trying to close a token (%p) ' \
290
- 'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
291
- end
292
- if @opened.pop
293
- @out << '</span>'
294
- @last_opened = @opened.last if @last_opened
295
- end
239
+ check_group_nesting 'token group', kind if $CODERAY_DEBUG
240
+ close_span
296
241
  end
297
242
 
298
243
  # whole lines to be highlighted, eg. a deleted line in a diff
299
244
  def begin_line kind
300
- if style = @span_for_kind[@last_opened ? [kind, *@opened] : kind]
245
+ if style = @span_for_kinds[@last_opened ? [kind, *@opened] : kind]
301
246
  if style['class="']
302
247
  @out << style.sub('class="', 'class="line ')
303
248
  else
@@ -311,16 +256,71 @@ module Encoders
311
256
  end
312
257
 
313
258
  def end_line kind
314
- if $CODERAY_DEBUG && (@opened.empty? || @opened.last != kind)
315
- warn 'Malformed token stream: Trying to close a line (%p) ' \
316
- 'that is not open. Open are: %p.' % [kind, @opened[1..-1]]
259
+ check_group_nesting 'line', kind if $CODERAY_DEBUG
260
+ close_span
261
+ end
262
+
263
+ protected
264
+
265
+ def check_options! options
266
+ unless [false, nil, :debug, :info, :info_long].include? options[:hint]
267
+ raise ArgumentError, "Unknown value %p for :hint; expected :info, :info_long, :debug, false, or nil." % [options[:hint]]
268
+ end
269
+
270
+ unless [:class, :style].include? options[:css]
271
+ raise ArgumentError, 'Unknown value %p for :css.' % [options[:css]]
272
+ end
273
+
274
+ options[:break_lines] = true if options[:line_numbers] == :inline
275
+ end
276
+
277
+ def css_class_for_kinds kinds
278
+ TokenKinds[kinds.is_a?(Symbol) ? kinds : kinds.first]
279
+ end
280
+
281
+ def style_for_kinds kinds
282
+ css_classes = kinds.is_a?(Array) ? kinds.map { |c| TokenKinds[c] } : [TokenKinds[kinds]]
283
+ @css.get_style_for_css_classes css_classes
284
+ end
285
+
286
+ def make_span_for_kinds method, hint
287
+ Hash.new do |h, kinds|
288
+ h[kinds.is_a?(Symbol) ? kinds : kinds.dup] = begin
289
+ css_class = css_class_for_kinds(kinds)
290
+ title = HTML.token_path_to_hint hint, kinds if hint
291
+
292
+ if css_class || title
293
+ if method == :style
294
+ style = style_for_kinds(kinds)
295
+ "<span#{title}#{" style=\"#{style}\"" if style}>"
296
+ else
297
+ "<span#{title}#{" class=\"#{css_class}\"" if css_class}>"
298
+ end
299
+ end
300
+ end
301
+ end
302
+ end
303
+
304
+ def check_group_nesting name, kind
305
+ if @opened.empty? || @opened.last != kind
306
+ warn "Malformed token stream: Trying to close a #{name} (%p) that is not open. Open are: %p." % [kind, @opened[1..-1]]
317
307
  end
308
+ end
309
+
310
+ def break_lines text, style
311
+ reopen = ''
312
+ @opened.each_with_index do |k, index|
313
+ reopen << (@span_for_kinds[index > 0 ? [k, *@opened[0...index]] : k] || '<span>')
314
+ end
315
+ text.gsub("\n", "#{'</span>' * @opened.size}#{'</span>' if style}\n#{reopen}#{style}")
316
+ end
317
+
318
+ def close_span
318
319
  if @opened.pop
319
320
  @out << '</span>'
320
321
  @last_opened = @opened.last if @last_opened
321
322
  end
322
323
  end
323
-
324
324
  end
325
325
 
326
326
  end
@@ -11,7 +11,7 @@ module Encoders
11
11
  end
12
12
 
13
13
  def initialize style = :default
14
- @classes = Hash.new
14
+ @styles = Hash.new
15
15
  style = CSS.load_stylesheet style
16
16
  @stylesheet = [
17
17
  style::CSS_MAIN_STYLES,
@@ -20,12 +20,12 @@ module Encoders
20
20
  parse style::TOKEN_COLORS
21
21
  end
22
22
 
23
- def get_style styles
24
- cl = @classes[styles.first]
23
+ def get_style_for_css_classes css_classes
24
+ cl = @styles[css_classes.first]
25
25
  return '' unless cl
26
26
  style = ''
27
- 1.upto styles.size do |offset|
28
- break if style = cl[styles[offset .. -1]]
27
+ 1.upto css_classes.size do |offset|
28
+ break if style = cl[css_classes[offset .. -1]]
29
29
  end
30
30
  # warn 'Style not found: %p' % [styles] if style.empty?
31
31
  return style
@@ -52,8 +52,8 @@ module Encoders
52
52
  for selector in selectors.split(',')
53
53
  classes = selector.scan(/[-\w]+/)
54
54
  cl = classes.pop
55
- @classes[cl] ||= Hash.new
56
- @classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
55
+ @styles[cl] ||= Hash.new
56
+ @styles[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
57
57
  end
58
58
  end
59
59
  end