rouge 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. data/Gemfile +2 -0
  2. data/lib/rouge/cli.rb +15 -0
  3. data/lib/rouge/demos/matlab +6 -0
  4. data/lib/rouge/demos/ocaml +12 -0
  5. data/lib/rouge/demos/scala +3 -0
  6. data/lib/rouge/demos/sml +4 -0
  7. data/lib/rouge/demos/vb +4 -0
  8. data/lib/rouge/formatters/html.rb +13 -17
  9. data/lib/rouge/formatters/terminal256.rb +2 -2
  10. data/lib/rouge/lexer.rb +12 -14
  11. data/lib/rouge/lexers/coffeescript.rb +2 -4
  12. data/lib/rouge/lexers/common_lisp.rb +2 -4
  13. data/lib/rouge/lexers/erlang.rb +4 -13
  14. data/lib/rouge/lexers/factor.rb +25 -24
  15. data/lib/rouge/lexers/gherkin.rb +6 -6
  16. data/lib/rouge/lexers/groovy.rb +1 -2
  17. data/lib/rouge/lexers/haml.rb +2 -2
  18. data/lib/rouge/lexers/haskell.rb +18 -20
  19. data/lib/rouge/lexers/http.rb +13 -12
  20. data/lib/rouge/lexers/ini.rb +1 -2
  21. data/lib/rouge/lexers/java.rb +1 -2
  22. data/lib/rouge/lexers/lua.rb +1 -1
  23. data/lib/rouge/lexers/make.rb +4 -5
  24. data/lib/rouge/lexers/markdown.rb +4 -12
  25. data/lib/rouge/lexers/matlab.rb +71 -0
  26. data/lib/rouge/lexers/matlab/builtins.rb +10 -0
  27. data/lib/rouge/lexers/nginx.rb +3 -4
  28. data/lib/rouge/lexers/objective_c.rb +1 -1
  29. data/lib/rouge/lexers/ocaml.rb +109 -0
  30. data/lib/rouge/lexers/perl.rb +2 -5
  31. data/lib/rouge/lexers/php.rb +5 -5
  32. data/lib/rouge/lexers/php/builtins.rb +4 -2
  33. data/lib/rouge/lexers/puppet.rb +1 -2
  34. data/lib/rouge/lexers/python.rb +5 -11
  35. data/lib/rouge/lexers/racket.rb +1 -3
  36. data/lib/rouge/lexers/ruby.rb +8 -8
  37. data/lib/rouge/lexers/rust.rb +3 -3
  38. data/lib/rouge/lexers/sass/common.rb +4 -4
  39. data/lib/rouge/lexers/scala.rb +141 -0
  40. data/lib/rouge/lexers/scheme.rb +1 -3
  41. data/lib/rouge/lexers/sed.rb +4 -4
  42. data/lib/rouge/lexers/shell.rb +1 -2
  43. data/lib/rouge/lexers/smalltalk.rb +6 -7
  44. data/lib/rouge/lexers/sml.rb +236 -6
  45. data/lib/rouge/lexers/tex.rb +1 -4
  46. data/lib/rouge/lexers/toml.rb +1 -2
  47. data/lib/rouge/lexers/vb.rb +162 -0
  48. data/lib/rouge/lexers/viml.rb +1 -1
  49. data/lib/rouge/lexers/yaml.rb +10 -11
  50. data/lib/rouge/regex_lexer.rb +77 -92
  51. data/lib/rouge/token.rb +3 -3
  52. data/lib/rouge/util.rb +4 -4
  53. data/lib/rouge/version.rb +1 -1
  54. metadata +13 -4
data/Gemfile CHANGED
@@ -7,6 +7,8 @@ gem 'wrong'
7
7
 
8
8
  gem 'rake'
9
9
 
10
+ gem 'racc'
11
+
10
12
  # don't try to install redcarpet under jruby
11
13
  gem 'redcarpet', :platforms => :ruby
12
14
 
@@ -42,6 +42,7 @@ module Rouge
42
42
  yield %| help #{Help.desc}|
43
43
  yield %| style #{Style.desc}|
44
44
  yield %| list #{List.desc}|
45
+ yield %| version #{Version.desc}|
45
46
  yield %||
46
47
  yield %|See `rougify help <command>` for more info.|
47
48
  end
@@ -80,6 +81,8 @@ module Rouge
80
81
 
81
82
  def self.class_from_arg(arg)
82
83
  case arg
84
+ when 'version', '--version'
85
+ Version
83
86
  when 'help'
84
87
  Help
85
88
  when 'highlight', 'hi'
@@ -91,6 +94,18 @@ module Rouge
91
94
  end
92
95
  end
93
96
 
97
+ class Version < CLI
98
+ def self.desc
99
+ "print the rouge version number"
100
+ end
101
+
102
+ def self.parse(*); new; end
103
+
104
+ def run
105
+ puts Rouge.version
106
+ end
107
+ end
108
+
94
109
  class Help < CLI
95
110
  def self.desc
96
111
  "print help info"
@@ -0,0 +1,6 @@
1
+ A = cat( 3, [1 2 3; 9 8 7; 4 6 5], [0 3 2; 8 8 4; 5 3 5], ...
2
+ [6 4 7; 6 8 5; 5 4 3]);
3
+ % The EIG function is applied to each of the horizontal 'slices' of A.
4
+ for i = 1:3
5
+ eig(squeeze(A(i,:,:)))
6
+ end
@@ -0,0 +1,12 @@
1
+ (* Binary tree with leaves car­rying an integer. *)
2
+ type tree = Leaf of int | Node of tree * tree
3
+
4
+ let rec exists_leaf test tree =
5
+ match tree with
6
+ | Leaf v -> test v
7
+ | Node (left, right) ->
8
+ exists_leaf test left
9
+ || exists_leaf test right
10
+
11
+ let has_even_leaf tree =
12
+ exists_leaf (fun n -> n mod 2 = 0) tree
@@ -0,0 +1,3 @@
1
+ class Greeter(name: String = "World") {
2
+ def sayHi() { println("Hi " + name + "!") }
3
+ }
@@ -0,0 +1,4 @@
1
+ datatype shape
2
+ = Circle of loc * real (* center and radius *)
3
+ | Square of loc * real (* upper-left corner and side length; axis-aligned *)
4
+ | Triangle of loc * loc * loc (* corners *)
@@ -0,0 +1,4 @@
1
+ Private Sub Form_Load()
2
+ ' Execute a simple message box that says "Hello, World!"
3
+ MsgBox "Hello, World!"
4
+ End Sub
@@ -46,7 +46,7 @@ module Rouge
46
46
  yield '</pre>' if @wrap
47
47
  end
48
48
 
49
- def stream_tableized(tokens, &b)
49
+ def stream_tableized(tokens)
50
50
  num_lines = 0
51
51
  last_val = ''
52
52
  formatted = ''
@@ -86,30 +86,26 @@ module Rouge
86
86
  yield '</div>' if @wrap
87
87
  end
88
88
 
89
- def span(tok, val, &b)
90
- # TODO: properly html-encode val
91
- val = CGI.escape_html(val)
89
+ TABLE_FOR_ESCAPE_HTML = {
90
+ '&' => '&amp;',
91
+ '<' => '&lt;',
92
+ '>' => '&gt;',
93
+ }
92
94
 
93
- case tok.shortname
94
- when ''
95
+ def span(tok, val)
96
+ val = val.gsub(/[&<>]/, TABLE_FOR_ESCAPE_HTML)
97
+ shortname = tok.shortname or raise "unknown token: #{tok.inspect} for #{val.inspect}"
98
+
99
+ if shortname.empty?
95
100
  yield val
96
- when nil
97
- raise "unknown token: #{tok.inspect} for #{val.inspect}"
98
101
  else
99
102
  if @inline_theme
100
103
  rules = @inline_theme.style_for(tok).rendered_rules
101
104
 
102
- yield '<span style='
103
- yield rules.to_a.join(';').inspect
104
- yield '>'
105
+ yield "<span style=\"#{rules.to_a.join(';')}\">#{val}</span>"
105
106
  else
106
- yield '<span class='
107
- yield tok.shortname.inspect
108
- yield '>'
107
+ yield "<span class=\"#{shortname}\">#{val}</span>"
109
108
  end
110
-
111
- yield val
112
- yield '</span>'
113
109
  end
114
110
  end
115
111
  end
@@ -83,7 +83,7 @@ module Rouge
83
83
  attrs = []
84
84
 
85
85
  attrs << ['38', '5', fg.to_s] if fg
86
- attrs << ['45', '5', bg.to_s] if bg
86
+ attrs << ['48', '5', bg.to_s] if bg
87
87
  attrs << '01' if style[:bold]
88
88
  attrs << '04' if style[:italic] # underline, but hey, whatevs
89
89
  escape(attrs)
@@ -155,7 +155,7 @@ module Rouge
155
155
  end
156
156
 
157
157
  def get_style(token)
158
- return text_style if token == Token::Tokens::Text
158
+ return text_style if token.ancestors.include? Token::Tokens::Text
159
159
 
160
160
  theme.get_own_style(token) || text_style
161
161
  end
@@ -322,6 +322,8 @@ module Rouge
322
322
  # tried and each stream consumed. Try it, it's pretty useful.
323
323
  def initialize(opts={})
324
324
  options(opts)
325
+
326
+ @debug = option(:debug)
325
327
  end
326
328
 
327
329
  # get and/or specify the options for this lexer.
@@ -340,25 +342,21 @@ module Rouge
340
342
  end
341
343
  end
342
344
 
345
+ # @deprecated
346
+ # Instead of `debug { "foo" }`, simply `puts "foo" if @debug`.
347
+ #
343
348
  # Leave a debug message if the `:debug` option is set. The message
344
349
  # is given as a block because some debug messages contain calculated
345
350
  # information that is unnecessary for lexing in the real world.
346
351
  #
352
+ # Calls to this method should be guarded with "if @debug" for best
353
+ # performance when debugging is turned off.
354
+ #
347
355
  # @example
348
- # debug { "hello, world!" }
349
- def debug(&b)
350
- # This method is a hotspot, unfortunately.
351
- #
352
- # For performance reasons, the "debug" option of a lexer cannot
353
- # be changed once it has begun lexing. This method will redefine
354
- # itself on the first call to a noop if "debug" is not set.
355
- if option(:debug)
356
- def self.debug; puts yield; end
357
- else
358
- def self.debug; end
359
- end
360
-
361
- debug(&b)
356
+ # debug { "hello, world!" } if @debug
357
+ def debug
358
+ warn "Lexer#debug is deprecated. Simply puts if @debug instead."
359
+ puts yield if @debug
362
360
  end
363
361
 
364
362
  # @abstract
@@ -80,14 +80,12 @@ module Rouge
80
80
  rule /[-=]>/, Name::Function
81
81
 
82
82
  rule /(@)([ \t]*)(#{id})/ do
83
- group Name::Variable::Instance; group Text
84
- group Name::Attribute
83
+ groups Name::Variable::Instance, Text, Name::Attribute
85
84
  push :slash_starts_regex
86
85
  end
87
86
 
88
87
  rule /([.])([ \t]*)(#{id})/ do
89
- group Punctuation; group Text
90
- group Name::Attribute
88
+ groups Punctuation, Text, Name::Attribute
91
89
  push :slash_starts_regex
92
90
  end
93
91
 
@@ -264,15 +264,13 @@ module Rouge
264
264
 
265
265
  # complex
266
266
  rule /(#c)(\()/i do
267
- group Num
268
- group Punctuation
267
+ groups Num, Punctuation
269
268
  push :root
270
269
  end
271
270
 
272
271
  # arrays and structures
273
272
  rule /(#(?:\d+a|s))(\()/i do
274
- group Literal::Other
275
- group Punctuation
273
+ groups Literal::Other, Punctuation
276
274
  push :root
277
275
  end
278
276
 
@@ -75,13 +75,10 @@ module Rouge
75
75
  rule(/<</, Name::Label)
76
76
  rule(/>>/, Name::Label)
77
77
  rule %r{(#{atom_re})(:)} do
78
- group Name::Namespace
79
- group Punctuation
78
+ groups Name::Namespace, Punctuation
80
79
  end
81
80
  rule %r{(?:^|(?<=:))(#{atom_re})(\s*)(\()} do
82
- group Name::Function
83
- group Text
84
- group Punctuation
81
+ groups Name::Function, Text, Punctuation
85
82
  end
86
83
  rule(%r{[+-]?#{base_re}#[0-9a-zA-Z]+}, Num::Integer)
87
84
  rule(/[+-]?\d+/, Num::Integer)
@@ -104,17 +101,11 @@ module Rouge
104
101
 
105
102
  state :directive do
106
103
  rule %r{(define)(\s*)(\()(#{macro_re})} do
107
- group Name::Entity
108
- group Text
109
- group Punctuation
110
- group Name::Constant
104
+ groups Name::Entity, Text, Punctuation, Name::Constant
111
105
  pop!
112
106
  end
113
107
  rule %r{(record)(\s*)(\()(#{macro_re})} do
114
- group Name::Entity
115
- group Text
116
- group Punctuation
117
- group Name::Label
108
+ groups Name::Entity, Text, Punctuation, Name::Label
118
109
  pop!
119
110
  end
120
111
  rule(atom_re, Name::Entity, :pop!)
@@ -166,64 +166,65 @@ module Rouge
166
166
  rule /\s+/m, Text
167
167
 
168
168
  rule /(:|::|MACRO:|MEMO:|GENERIC:|HELP:)(\s+)(\S+)/m do
169
- group Keyword; group Text
170
- group Name::Function
169
+ groups Keyword, Text, Name::Function
171
170
  end
172
171
 
173
172
  rule /(M:|HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)/m do
174
- group Keyword; group Text
175
- group Name::Class; group Text
176
- group Name::Function
173
+ groups Keyword, Text, Name::Class, Text, Name::Function
177
174
  end
178
175
 
179
176
  rule /\((?=\s)/, Name::Function, :stack_effect
180
177
  rule /;(?=\s)/, Keyword
181
178
 
182
179
  rule /(USING:)((?:\s|\\\s)+)/m do
183
- group Keyword::Namespace; group Text
180
+ groups Keyword::Namespace, Text
184
181
  push :import
185
182
  end
186
183
 
187
184
  rule /(IN:|USE:|UNUSE:|QUALIFIED:|QUALIFIED-WITH:)(\s+)(\S+)/m do
188
- group Keyword::Namespace; group Text; group Name::Namespace
185
+ groups Keyword::Namespace, Text, Name::Namespace
189
186
  end
190
187
 
191
188
  rule /(FROM:|EXCLUDE:)(\s+)(\S+)(\s+)(=>)/m do
192
- group Keyword::Namespace; group Text
193
- group Name::Namespace; group Text
194
- group Punctuation
189
+ groups Keyword::Namespace, Text, Name::Namespace, Text, Punctuation
195
190
  end
196
191
 
197
192
  rule /(?:ALIAS|DEFER|FORGET|POSTPONE):/, Keyword::Namespace
198
193
 
199
194
  rule /(TUPLE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
200
- group Keyword; group Text
201
- group Name::Class; group Text
202
- group Punctuation; group Text
203
- group Name::Class
195
+ groups(
196
+ Keyword, Text,
197
+ Name::Class, Text,
198
+ Punctuation, Text,
199
+ Name::Class
200
+ )
204
201
  push :slots
205
202
  end
206
203
 
207
204
  rule /(TUPLE:)(\s+)(\S+)/m do
208
- group Keyword; group Text; group Name::Class
205
+ groups Keyword, Text, Name::Class
209
206
  push :slots
210
207
  end
211
208
 
212
209
  rule /(UNION:|INTERSECTION:)(\s+)(\S+)/m do
213
- group Keyword; group Text; group Name::Class
210
+ groups Keyword, Text, Name::Class
214
211
  end
215
212
 
216
213
  rule /(PREDICATE:)(\s+)(\S+)(\s+)(<)(\s+)(\S+)/m do
217
- group Keyword; group Text
218
- group Name::Class; group Text
219
- group Punctuation; group Text
220
- group Name::Class
214
+ groups(
215
+ Keyword, Text,
216
+ Name::Class, Text,
217
+ Punctuation, Text,
218
+ Name::Class
219
+ )
221
220
  end
222
221
 
223
222
  rule /(C:)(\s+)(\S+)(\s+)(\S+)/m do
224
- group Keyword; group Text
225
- group Name::Function; group Text
226
- group Name::Class
223
+ groups(
224
+ Keyword, Text,
225
+ Name::Function, Text,
226
+ Name::Class
227
+ )
227
228
  end
228
229
 
229
230
  rule %r(
@@ -234,7 +235,7 @@ module Rouge
234
235
  rule /(?:<PRIVATE|PRIVATE>)/, Keyword::Namespace
235
236
 
236
237
  rule /(MAIN:)(\s+)(\S+)/ do
237
- group Keyword::Namespace; group Text; group Name::Function
238
+ groups Keyword::Namespace, Text, Name::Function
238
239
  end
239
240
 
240
241
  # strings
@@ -54,17 +54,17 @@ module Rouge
54
54
  reset_stack
55
55
 
56
56
  keyword = m[1]
57
- if self.class.keywords[:element].include? keyword
58
- group Keyword::Namespace; push :description
57
+ keyword_tok = if self.class.keywords[:element].include? keyword
58
+ push :description; Keyword::Namespace
59
59
  elsif self.class.keywords[:feature].include? keyword
60
- group Keyword::Declaration; push :feature_description
60
+ push :feature_description; Keyword::Declaration
61
61
  elsif self.class.keywords[:examples].include? keyword
62
- group Name::Namespace; push :example_description
62
+ push :example_description; Name::Namespace
63
63
  else
64
- group Error
64
+ Error
65
65
  end
66
66
 
67
- group Punctuation
67
+ groups keyword_tok, Punctuation
68
68
  end
69
69
  end
70
70
 
@@ -60,8 +60,7 @@ module Rouge
60
60
  rule %r(/(\\\\|\\"|[^/])*/), Str
61
61
  rule /'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'/, Str::Char
62
62
  rule /(\.)([a-zA-Z_][a-zA-Z0-9_]*)/ do
63
- group Operator
64
- group Name::Attribute
63
+ groups Operator, Name::Attribute
65
64
  end
66
65
 
67
66
  rule /[a-zA-Z_][a-zA-Z0-9_]*:/, Name::Label
@@ -82,7 +82,7 @@ module Rouge
82
82
  rule %r(
83
83
  (/) (\[#{dot}*?\]) (#{dot}*\n)
84
84
  )x do
85
- group Comment; group Comment::Special; group Comment
85
+ groups Comment, Comment::Special, Comment
86
86
  pop!
87
87
  end
88
88
 
@@ -115,7 +115,7 @@ module Rouge
115
115
  @filter_lexer = self.filters[filter_name]
116
116
  @filter_lexer.reset! unless @filter_lexer.nil?
117
117
 
118
- debug { " haml: filter #{filter_name.inspect} #{@filter_lexer.inspect}" }
118
+ puts " haml: filter #{filter_name.inspect} #{@filter_lexer.inspect}" if @debug
119
119
  end
120
120
 
121
121
  mixin :eval_or_plain
@@ -88,32 +88,32 @@ module Rouge
88
88
  rule /\bqualified\b/, Keyword
89
89
  # import X as Y
90
90
  rule /([A-Z][\w.]*)(\s+)(as)(\s+)([A-Z][a-zA-Z0-9_.]*)/ do
91
- group Name::Namespace # X
92
- group Text
93
- group Keyword # as
94
- group Text
95
- group Name # Y
91
+ groups(
92
+ Name::Namespace, # X
93
+ Text, Keyword, # as
94
+ Text, Name # Y
95
+ )
96
96
  pop!
97
97
  end
98
98
 
99
99
  # import X hiding (functions)
100
100
  rule /([A-Z][\w.]*)(\s+)(hiding)(\s+)(\()/ do
101
- group Name::Namespace # X
102
- group Text
103
- group Keyword # hiding
104
- group Text
105
- group Punctuation # (
106
- pop!
107
- push :funclist
101
+ groups(
102
+ Name::Namespace, # X
103
+ Text, Keyword, # hiding
104
+ Text, Punctuation # (
105
+ )
106
+ goto :funclist
108
107
  end
109
108
 
110
109
  # import X (functions)
111
110
  rule /([A-Z][\w.]*)(\s+)(\()/ do
112
- group Name::Namespace # X
113
- group Text
114
- group Punctuation # (
115
- pop!
116
- push :funclist
111
+ groups(
112
+ Name::Namespace, # X
113
+ Text,
114
+ Punctuation # (
115
+ )
116
+ goto :funclist
117
117
  end
118
118
 
119
119
  rule /[\w.]+/, Name::Namespace, :pop!
@@ -123,9 +123,7 @@ module Rouge
123
123
  rule /\s+/, Text
124
124
  # module Foo (functions)
125
125
  rule /([A-Z][\w.]*)(\s+)(\()/ do
126
- group Name::Namespace
127
- group Text
128
- group Punctuation
126
+ groups Name::Namespace, Text, Punctuation
129
127
  push :funclist
130
128
  end
131
129