coderay 1.0.9 → 1.1.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/Rakefile +2 -0
  3. data/bin/coderay +4 -4
  4. data/lib/coderay.rb +2 -3
  5. data/lib/coderay/encoders/debug.rb +5 -17
  6. data/lib/coderay/encoders/debug_lint.rb +62 -0
  7. data/lib/coderay/encoders/html.rb +84 -84
  8. data/lib/coderay/encoders/html/css.rb +7 -7
  9. data/lib/coderay/encoders/html/numbering.rb +24 -19
  10. data/lib/coderay/encoders/html/output.rb +1 -1
  11. data/lib/coderay/encoders/lint.rb +57 -0
  12. data/lib/coderay/encoders/statistic.rb +0 -1
  13. data/lib/coderay/encoders/terminal.rb +121 -105
  14. data/lib/coderay/helpers/file_type.rb +54 -47
  15. data/lib/coderay/helpers/plugin.rb +4 -13
  16. data/lib/coderay/scanner.rb +58 -26
  17. data/lib/coderay/scanners/c.rb +1 -1
  18. data/lib/coderay/scanners/cpp.rb +1 -1
  19. data/lib/coderay/scanners/css.rb +22 -25
  20. data/lib/coderay/scanners/diff.rb +53 -31
  21. data/lib/coderay/scanners/groovy.rb +17 -4
  22. data/lib/coderay/scanners/html.rb +38 -16
  23. data/lib/coderay/scanners/java.rb +1 -1
  24. data/lib/coderay/scanners/java_script.rb +30 -6
  25. data/lib/coderay/scanners/json.rb +15 -12
  26. data/lib/coderay/scanners/lua.rb +280 -0
  27. data/lib/coderay/scanners/php.rb +22 -4
  28. data/lib/coderay/scanners/python.rb +3 -3
  29. data/lib/coderay/scanners/raydebug.rb +8 -8
  30. data/lib/coderay/scanners/ruby.rb +2 -2
  31. data/lib/coderay/scanners/sass.rb +232 -0
  32. data/lib/coderay/scanners/sql.rb +7 -4
  33. data/lib/coderay/scanners/taskpaper.rb +36 -0
  34. data/lib/coderay/scanners/yaml.rb +2 -2
  35. data/lib/coderay/styles/alpha.rb +31 -21
  36. data/lib/coderay/token_kinds.rb +68 -71
  37. data/lib/coderay/tokens.rb +23 -77
  38. data/lib/coderay/version.rb +1 -1
  39. data/test/functional/examples.rb +3 -3
  40. data/test/functional/for_redcloth.rb +4 -10
  41. metadata +13 -14
  42. data/lib/coderay/helpers/gzip.rb +0 -41
@@ -265,7 +265,7 @@ module Scanners
265
265
  @html_scanner.tokenize match unless match.empty?
266
266
  end
267
267
 
268
- when :php
268
+ when :php, :php_inline
269
269
  if match = scan(/\s+/)
270
270
  encoder.text_token match, :space
271
271
 
@@ -332,7 +332,7 @@ module Scanners
332
332
  if states.size == 1
333
333
  encoder.text_token match, :error
334
334
  else
335
- states.pop
335
+ state = states.pop
336
336
  if states.last.is_a?(::Array)
337
337
  delimiter = states.last[1]
338
338
  states[-1] = states.last[0]
@@ -340,6 +340,7 @@ module Scanners
340
340
  encoder.end_group :inline
341
341
  else
342
342
  encoder.text_token match, :operator
343
+ encoder.end_group :inline if state == :php_inline
343
344
  label_expected = true
344
345
  end
345
346
  end
@@ -350,7 +351,14 @@ module Scanners
350
351
 
351
352
  elsif match = scan(RE::PHP_END)
352
353
  encoder.text_token match, :inline_delimiter
353
- states = [:initial]
354
+ while state = states.pop
355
+ encoder.end_group :string if [:sqstring, :dqstring].include? state
356
+ if state.is_a? Array
357
+ encoder.end_group :inline
358
+ encoder.end_group :string if [:sqstring, :dqstring].include? state.first
359
+ end
360
+ end
361
+ states << :initial
354
362
 
355
363
  elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
356
364
  encoder.begin_group :string
@@ -400,6 +408,7 @@ module Scanners
400
408
  elsif match = scan(/\\/)
401
409
  encoder.text_token match, :error
402
410
  else
411
+ encoder.end_group :string
403
412
  states.pop
404
413
  end
405
414
 
@@ -459,7 +468,7 @@ module Scanners
459
468
  encoder.begin_group :inline
460
469
  states[-1] = [states.last, delimiter]
461
470
  delimiter = nil
462
- states.push :php
471
+ states.push :php_inline
463
472
  encoder.text_token match, :delimiter
464
473
  else
465
474
  encoder.text_token match, :content
@@ -469,6 +478,7 @@ module Scanners
469
478
  elsif match = scan(/\$/)
470
479
  encoder.text_token match, :content
471
480
  else
481
+ encoder.end_group :string
472
482
  states.pop
473
483
  end
474
484
 
@@ -500,6 +510,14 @@ module Scanners
500
510
 
501
511
  end
502
512
 
513
+ while state = states.pop
514
+ encoder.end_group :string if [:sqstring, :dqstring].include? state
515
+ if state.is_a? Array
516
+ encoder.end_group :inline
517
+ encoder.end_group :string if [:sqstring, :dqstring].include? state.first
518
+ end
519
+ end
520
+
503
521
  encoder
504
522
  end
505
523
 
@@ -133,7 +133,7 @@ module Scanners
133
133
  elsif match = scan(/ \\ | $ /x)
134
134
  encoder.end_group string_type
135
135
  string_type = nil
136
- encoder.text_token match, :error
136
+ encoder.text_token match, :error unless match.empty?
137
137
  state = :initial
138
138
  else
139
139
  raise_inspect "else case \" reached; %p not handled." % peek(1), encoder, state
@@ -157,12 +157,12 @@ module Scanners
157
157
  encoder.text_token match, :operator
158
158
 
159
159
  elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
160
+ modifiers = self[1]
160
161
  string_delimiter = self[2]
161
- string_type = docstring_coming ? :docstring : :string
162
+ string_type = docstring_coming ? :docstring : (modifiers == 'b' ? :binary : :string)
162
163
  docstring_coming = false if docstring_coming
163
164
  encoder.begin_group string_type
164
165
  string_raw = false
165
- modifiers = self[1]
166
166
  unless modifiers.empty?
167
167
  string_raw = !!modifiers.index(?r)
168
168
  encoder.text_token modifiers, :modifier
@@ -1,11 +1,11 @@
1
1
  module CodeRay
2
2
  module Scanners
3
-
3
+
4
4
  # = Debug Scanner
5
5
  #
6
6
  # Parses the output of the Encoders::Debug encoder.
7
7
  class Raydebug < Scanner
8
-
8
+
9
9
  register_for :raydebug
10
10
  file_extension 'raydebug'
11
11
  title 'CodeRay Token Dump'
@@ -13,11 +13,11 @@ module Scanners
13
13
  protected
14
14
 
15
15
  def scan_tokens encoder, options
16
-
16
+
17
17
  opened_tokens = []
18
-
18
+
19
19
  until eos?
20
-
20
+
21
21
  if match = scan(/\s+/)
22
22
  encoder.text_token match, :space
23
23
 
@@ -26,7 +26,7 @@ module Scanners
26
26
  encoder.text_token kind, :class
27
27
  encoder.text_token '(', :operator
28
28
  match = self[2]
29
- encoder.text_token match, kind.to_sym
29
+ encoder.text_token match, kind.to_sym unless match.empty?
30
30
  encoder.text_token match, :operator if match = scan(/\)/)
31
31
 
32
32
  elsif match = scan(/ (\w+) ([<\[]) /x)
@@ -59,8 +59,8 @@ module Scanners
59
59
 
60
60
  encoder
61
61
  end
62
-
62
+
63
63
  end
64
-
64
+
65
65
  end
66
66
  end
@@ -96,7 +96,7 @@ module Scanners
96
96
  /#{patterns::METHOD_NAME}/o)
97
97
 
98
98
  kind = patterns::IDENT_KIND[match]
99
- if kind == :ident && value_expected != :colon_expected && scan(/:(?!:)/)
99
+ if value_expected != :colon_expected && scan(/:(?!:)/)
100
100
  value_expected = true
101
101
  encoder.text_token match, :key
102
102
  encoder.text_token ':', :operator
@@ -269,7 +269,7 @@ module Scanners
269
269
  end
270
270
 
271
271
  if last_state
272
- state = last_state
272
+ state = last_state unless state.is_a?(StringState) # otherwise, a simple 'def"' results in unclosed tokens
273
273
  last_state = nil
274
274
  end
275
275
 
@@ -0,0 +1,232 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ # A scanner for Sass.
5
+ class Sass < CSS
6
+
7
+ register_for :sass
8
+ file_extension 'sass'
9
+
10
+ protected
11
+
12
+ def setup
13
+ @state = :initial
14
+ end
15
+
16
+ def scan_tokens encoder, options
17
+ states = Array(options[:state] || @state).dup
18
+
19
+ encoder.begin_group :string if states.last == :sqstring || states.last == :dqstring
20
+
21
+ until eos?
22
+
23
+ if bol? && (match = scan(/(?>( +)?(\/[\*\/])(.+)?)(?=\n)/))
24
+ encoder.text_token self[1], :space if self[1]
25
+ encoder.begin_group :comment
26
+ encoder.text_token self[2], :delimiter
27
+ encoder.text_token self[3], :content if self[3]
28
+ if match = scan(/(?:\n+#{self[1]} .*)+/)
29
+ encoder.text_token match, :content
30
+ end
31
+ encoder.end_group :comment
32
+ elsif match = scan(/\n|[^\n\S]+\n?/)
33
+ encoder.text_token match, :space
34
+ if match.index(/\n/)
35
+ value_expected = false
36
+ states.pop if states.last == :include
37
+ end
38
+
39
+ elsif states.last == :sass_inline && (match = scan(/\}/))
40
+ encoder.text_token match, :inline_delimiter
41
+ encoder.end_group :inline
42
+ states.pop
43
+
44
+ elsif case states.last
45
+ when :initial, :media, :sass_inline
46
+ if match = scan(/(?>#{RE::Ident})(?!\()/ox)
47
+ encoder.text_token match, value_expected ? :value : (check(/.*:(?![a-z])/) ? :key : :tag)
48
+ next
49
+ elsif !value_expected && (match = scan(/\*/))
50
+ encoder.text_token match, :tag
51
+ next
52
+ elsif match = scan(RE::Class)
53
+ encoder.text_token match, :class
54
+ next
55
+ elsif match = scan(RE::Id)
56
+ encoder.text_token match, :id
57
+ next
58
+ elsif match = scan(RE::PseudoClass)
59
+ encoder.text_token match, :pseudo_class
60
+ next
61
+ elsif match = scan(RE::AttributeSelector)
62
+ # TODO: Improve highlighting inside of attribute selectors.
63
+ encoder.text_token match[0,1], :operator
64
+ encoder.text_token match[1..-2], :attribute_name if match.size > 2
65
+ encoder.text_token match[-1,1], :operator if match[-1] == ?]
66
+ next
67
+ elsif match = scan(/(\=|@mixin +)#{RE::Ident}/o)
68
+ encoder.text_token match, :function
69
+ next
70
+ elsif match = scan(/@import\b/)
71
+ encoder.text_token match, :directive
72
+ states << :include
73
+ next
74
+ elsif match = scan(/@media\b/)
75
+ encoder.text_token match, :directive
76
+ # states.push :media_before_name
77
+ next
78
+ end
79
+
80
+ when :block
81
+ if match = scan(/(?>#{RE::Ident})(?!\()/ox)
82
+ if value_expected
83
+ encoder.text_token match, :value
84
+ else
85
+ encoder.text_token match, :key
86
+ end
87
+ next
88
+ end
89
+
90
+ when :sqstring, :dqstring
91
+ if match = scan(states.last == :sqstring ? /(?:[^\n\'\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o : /(?:[^\n\"\#]+|\\\n|#{RE::Escape}|#(?!\{))+/o)
92
+ encoder.text_token match, :content
93
+ elsif match = scan(/['"]/)
94
+ encoder.text_token match, :delimiter
95
+ encoder.end_group :string
96
+ states.pop
97
+ elsif match = scan(/#\{/)
98
+ encoder.begin_group :inline
99
+ encoder.text_token match, :inline_delimiter
100
+ states.push :sass_inline
101
+ elsif match = scan(/ \\ | $ /x)
102
+ encoder.end_group states.last
103
+ encoder.text_token match, :error unless match.empty?
104
+ states.pop
105
+ else
106
+ raise_inspect "else case #{states.last} reached; %p not handled." % peek(1), encoder
107
+ end
108
+
109
+ when :include
110
+ if match = scan(/[^\s'",]+/)
111
+ encoder.text_token match, :include
112
+ next
113
+ end
114
+
115
+ else
116
+ #:nocov:
117
+ raise_inspect 'Unknown state: %p' % [states.last], encoder
118
+ #:nocov:
119
+
120
+ end
121
+
122
+ elsif match = scan(/\$#{RE::Ident}/o)
123
+ encoder.text_token match, :variable
124
+ next
125
+
126
+ elsif match = scan(/&/)
127
+ encoder.text_token match, :local_variable
128
+
129
+ elsif match = scan(/\+#{RE::Ident}/o)
130
+ encoder.text_token match, :include
131
+ value_expected = true
132
+
133
+ elsif match = scan(/\/\*(?:.*?\*\/|.*)|\/\/.*/)
134
+ encoder.text_token match, :comment
135
+
136
+ elsif match = scan(/#\{/)
137
+ encoder.begin_group :inline
138
+ encoder.text_token match, :inline_delimiter
139
+ states.push :sass_inline
140
+
141
+ elsif match = scan(/\{/)
142
+ value_expected = false
143
+ encoder.text_token match, :operator
144
+ states.push :block
145
+
146
+ elsif match = scan(/\}/)
147
+ value_expected = false
148
+ encoder.text_token match, :operator
149
+ if states.last == :block || states.last == :media
150
+ states.pop
151
+ end
152
+
153
+ elsif match = scan(/['"]/)
154
+ encoder.begin_group :string
155
+ encoder.text_token match, :delimiter
156
+ if states.include? :sass_inline
157
+ # no nesting, just scan the string until delimiter
158
+ content = scan_until(/(?=#{match}|\}|\z)/)
159
+ encoder.text_token content, :content unless content.empty?
160
+ encoder.text_token match, :delimiter if scan(/#{match}/)
161
+ encoder.end_group :string
162
+ else
163
+ states.push match == "'" ? :sqstring : :dqstring
164
+ end
165
+
166
+ elsif match = scan(/#{RE::Function}/o)
167
+ encoder.begin_group :function
168
+ start = match[/^[-\w]+\(/]
169
+ encoder.text_token start, :delimiter
170
+ if match[-1] == ?)
171
+ encoder.text_token match[start.size..-2], :content
172
+ encoder.text_token ')', :delimiter
173
+ else
174
+ encoder.text_token match[start.size..-1], :content if start.size < match.size
175
+ end
176
+ encoder.end_group :function
177
+
178
+ elsif match = scan(/[a-z][-a-z_]*(?=\()/o)
179
+ encoder.text_token match, :predefined
180
+
181
+ elsif match = scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
182
+ encoder.text_token match, :float
183
+
184
+ elsif match = scan(/#{RE::HexColor}/o)
185
+ encoder.text_token match, :color
186
+
187
+ elsif match = scan(/! *(?:important|optional)/)
188
+ encoder.text_token match, :important
189
+
190
+ elsif match = scan(/(?:rgb|hsl)a?\([^()\n]*\)?/)
191
+ encoder.text_token match, :color
192
+
193
+ elsif match = scan(/@else if\b|#{RE::AtKeyword}/o)
194
+ encoder.text_token match, :directive
195
+ value_expected = true
196
+
197
+ elsif match = scan(/ == | != | [-+*\/>~:;,.=()] /x)
198
+ if match == ':'
199
+ value_expected = true
200
+ elsif match == ';'
201
+ value_expected = false
202
+ end
203
+ encoder.text_token match, :operator
204
+
205
+ else
206
+ encoder.text_token getch, :error
207
+
208
+ end
209
+
210
+ end
211
+
212
+ states.pop if states.last == :include
213
+
214
+ if options[:keep_state]
215
+ @state = states.dup
216
+ end
217
+
218
+ while state = states.pop
219
+ if state == :sass_inline
220
+ encoder.end_group :inline
221
+ elsif state == :sqstring || state == :dqstring
222
+ encoder.end_group :string
223
+ end
224
+ end
225
+
226
+ encoder
227
+ end
228
+
229
+ end
230
+
231
+ end
232
+ end
@@ -1,8 +1,9 @@
1
- module CodeRay module Scanners
1
+ module CodeRay
2
+ module Scanners
2
3
 
3
4
  # by Josh Goebel
4
5
  class SQL < Scanner
5
-
6
+
6
7
  register_for :sql
7
8
 
8
9
  KEYWORDS = %w(
@@ -148,7 +149,8 @@ module CodeRay module Scanners
148
149
  encoder.text_token string_content, :content
149
150
  string_content = ''
150
151
  end
151
- encoder.text_token match, :error
152
+ encoder.text_token match, :error unless match.empty?
153
+ encoder.end_group :string
152
154
  state = :initial
153
155
  else
154
156
  raise "else case \" reached; %p not handled." % peek(1), encoder
@@ -171,4 +173,5 @@ module CodeRay module Scanners
171
173
 
172
174
  end
173
175
 
174
- end end
176
+ end
177
+ end
@@ -0,0 +1,36 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class Taskpaper < Scanner
5
+
6
+ register_for :taskpaper
7
+ file_extension 'taskpaper'
8
+
9
+ protected
10
+
11
+ def scan_tokens encoder, options
12
+ until eos?
13
+ if match = scan(/\S.*:.*$/) # project
14
+ encoder.text_token(match, :namespace)
15
+ elsif match = scan(/-.+@done.*/) # completed task
16
+ encoder.text_token(match, :done)
17
+ elsif match = scan(/-(?:[^@\n]+|@(?!due))*/) # task
18
+ encoder.text_token(match, :plain)
19
+ elsif match = scan(/@due.*/) # comment
20
+ encoder.text_token(match, :important)
21
+ elsif match = scan(/.+/) # comment
22
+ encoder.text_token(match, :comment)
23
+ elsif match = scan(/\s+/) # space
24
+ encoder.text_token(match, :space)
25
+ else # other
26
+ encoder.text_token getch, :error
27
+ end
28
+ end
29
+
30
+ encoder
31
+ end
32
+
33
+ end
34
+
35
+ end
36
+ end