rouge 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +2 -0
  2. data/bin/rougify +1 -1
  3. data/lib/rouge/demos/{text → plaintext} +0 -0
  4. data/lib/rouge/formatters/html.rb +13 -8
  5. data/lib/rouge/formatters/terminal256.rb +15 -11
  6. data/lib/rouge/lexer.rb +3 -1
  7. data/lib/rouge/lexers/c.rb +65 -55
  8. data/lib/rouge/lexers/clojure.rb +19 -19
  9. data/lib/rouge/lexers/coffeescript.rb +42 -42
  10. data/lib/rouge/lexers/common_lisp.rb +49 -48
  11. data/lib/rouge/lexers/conf.rb +6 -6
  12. data/lib/rouge/lexers/cpp.rb +48 -48
  13. data/lib/rouge/lexers/csharp.rb +25 -25
  14. data/lib/rouge/lexers/css.rb +32 -33
  15. data/lib/rouge/lexers/diff.rb +8 -8
  16. data/lib/rouge/lexers/elixir.rb +30 -30
  17. data/lib/rouge/lexers/erb.rb +5 -5
  18. data/lib/rouge/lexers/erlang.rb +38 -38
  19. data/lib/rouge/lexers/factor.rb +56 -56
  20. data/lib/rouge/lexers/gherkin.rb +27 -27
  21. data/lib/rouge/lexers/go.rb +21 -21
  22. data/lib/rouge/lexers/groovy.rb +32 -32
  23. data/lib/rouge/lexers/haml.rb +35 -35
  24. data/lib/rouge/lexers/handlebars.rb +27 -27
  25. data/lib/rouge/lexers/haskell.rb +78 -79
  26. data/lib/rouge/lexers/html.rb +28 -28
  27. data/lib/rouge/lexers/http.rb +13 -13
  28. data/lib/rouge/lexers/ini.rb +13 -13
  29. data/lib/rouge/lexers/io.rb +19 -19
  30. data/lib/rouge/lexers/java.rb +29 -29
  31. data/lib/rouge/lexers/javascript.rb +49 -52
  32. data/lib/rouge/lexers/literate_haskell.rb +5 -5
  33. data/lib/rouge/lexers/llvm.rb +14 -14
  34. data/lib/rouge/lexers/lua.rb +33 -33
  35. data/lib/rouge/lexers/make.rb +26 -26
  36. data/lib/rouge/lexers/markdown.rb +49 -49
  37. data/lib/rouge/lexers/nginx.rb +19 -19
  38. data/lib/rouge/lexers/perl.rb +53 -54
  39. data/lib/rouge/lexers/php.rb +42 -43
  40. data/lib/rouge/lexers/{text.rb → plain_text.rb} +3 -2
  41. data/lib/rouge/lexers/prolog.rb +19 -19
  42. data/lib/rouge/lexers/puppet.rb +40 -40
  43. data/lib/rouge/lexers/python.rb +63 -63
  44. data/lib/rouge/lexers/r.rb +15 -15
  45. data/lib/rouge/lexers/racket.rb +29 -29
  46. data/lib/rouge/lexers/ruby.rb +94 -97
  47. data/lib/rouge/lexers/rust.rb +37 -37
  48. data/lib/rouge/lexers/sass.rb +15 -15
  49. data/lib/rouge/lexers/sass/common.rb +144 -136
  50. data/lib/rouge/lexers/scheme.rb +25 -25
  51. data/lib/rouge/lexers/scss.rb +7 -7
  52. data/lib/rouge/lexers/sed.rb +39 -39
  53. data/lib/rouge/lexers/shell.rb +45 -45
  54. data/lib/rouge/lexers/smalltalk.rb +36 -36
  55. data/lib/rouge/lexers/sql.rb +27 -27
  56. data/lib/rouge/lexers/tcl.rb +34 -34
  57. data/lib/rouge/lexers/tex.rb +24 -24
  58. data/lib/rouge/lexers/toml.rb +18 -18
  59. data/lib/rouge/lexers/viml.rb +19 -19
  60. data/lib/rouge/lexers/xml.rb +16 -16
  61. data/lib/rouge/lexers/yaml.rb +78 -81
  62. data/lib/rouge/plugins/redcarpet.rb +1 -1
  63. data/lib/rouge/regex_lexer.rb +31 -14
  64. data/lib/rouge/template_lexer.rb +1 -1
  65. data/lib/rouge/theme.rb +12 -10
  66. data/lib/rouge/themes/thankful_eyes.rb +43 -43
  67. data/lib/rouge/token.rb +145 -148
  68. data/lib/rouge/version.rb +1 -1
  69. metadata +4 -4
@@ -21,35 +21,35 @@ module Rouge
21
21
  end
22
22
 
23
23
  state :root do
24
- rule /[^<&]+/, 'Text'
25
- rule /&\S*?;/, 'Name.Entity'
26
- rule /<!\[CDATA\[.*?\]\]\>/, 'Comment.Preproc'
27
- rule /<!--/, 'Comment', :comment
28
- rule /<\?.*?\?>/, 'Comment.Preproc'
29
- rule /<![^>]*>/, 'Comment.Preproc'
24
+ rule /[^<&]+/, Text
25
+ rule /&\S*?;/, Name::Entity
26
+ rule /<!\[CDATA\[.*?\]\]\>/, Comment::Preproc
27
+ rule /<!--/, Comment, :comment
28
+ rule /<\?.*?\?>/, Comment::Preproc
29
+ rule /<![^>]*>/, Comment::Preproc
30
30
 
31
31
  # open tags
32
- rule %r(<\s*[\w:.-]+)m, 'Name.Tag', :tag
32
+ rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
33
33
 
34
34
  # self-closing tags
35
- rule %r(<\s*/\s*[\w:.-]+\s*>)m, 'Name.Tag'
35
+ rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
36
36
  end
37
37
 
38
38
  state :comment do
39
- rule /[^-]+/m, 'Comment'
40
- rule /-->/, 'Comment', :pop!
41
- rule /-/, 'Comment'
39
+ rule /[^-]+/m, Comment
40
+ rule /-->/, Comment, :pop!
41
+ rule /-/, Comment
42
42
  end
43
43
 
44
44
  state :tag do
45
- rule /\s+/m, 'Text'
46
- rule /[\w.:-]+\s*=/m, 'Name.Attribute', :attr
47
- rule %r(/?\s*>), 'Name.Tag', :pop!
45
+ rule /\s+/m, Text
46
+ rule /[\w.:-]+\s*=/m, Name::Attribute, :attr
47
+ rule %r(/?\s*>), Name::Tag, :pop!
48
48
  end
49
49
 
50
50
  state :attr do
51
- rule /\s+/m, 'Text'
52
- rule /".*?"|'.*?'|[^\s>]+/, 'Literal.String', :pop!
51
+ rule /\s+/m, Text
52
+ rule /".*?"|'.*?'|[^\s>]+/, Str, :pop!
53
53
  end
54
54
  end
55
55
  end
@@ -37,9 +37,7 @@ module Rouge
37
37
  end
38
38
 
39
39
  # Save a possible indentation level
40
- def save_indent(opts={})
41
- debug { " yaml: save_indent" }
42
- match = @last_match[0]
40
+ def save_indent(match)
43
41
  @next_indent = match.size
44
42
  debug { " yaml: indent: #{self.indent}/#@next_indent" }
45
43
  debug { " yaml: popping indent stack - before: #@indent_stack" }
@@ -55,9 +53,9 @@ module Rouge
55
53
  end
56
54
  end
57
55
 
58
- def continue_indent
56
+ def continue_indent(match)
59
57
  debug { " yaml: continue_indent" }
60
- @next_indent += @last_match[0].size
58
+ @next_indent += match.size
61
59
  end
62
60
 
63
61
  def set_indent(opts={})
@@ -73,65 +71,65 @@ module Rouge
73
71
  start { reset_indent }
74
72
 
75
73
  state :basic do
76
- rule /#.*$/, 'Comment.Single'
74
+ rule /#.*$/, Comment::Single
77
75
  end
78
76
 
79
77
  state :root do
80
78
  mixin :basic
81
79
 
82
- rule /\n+/, 'Text'
80
+ rule /\n+/, Text
83
81
 
84
82
  # trailing or pre-comment whitespace
85
- rule /[ ]+(?=#|$)/, 'Text'
83
+ rule /[ ]+(?=#|$)/, Text
86
84
 
87
85
  rule /^%YAML\b/ do
88
- token 'Name.Tag'
86
+ token Name::Tag
89
87
  reset_indent
90
88
  push :yaml_directive
91
89
  end
92
90
 
93
91
  rule /^%TAG\b/ do
94
- token 'Name.Tag'
92
+ token Name::Tag
95
93
  reset_indent
96
94
  push :tag_directive
97
95
  end
98
96
 
99
97
  # doc-start and doc-end indicators
100
98
  rule /^(?:---|\.\.\.)(?= |$)/ do
101
- token 'Name.Namespace'
99
+ token Name::Namespace
102
100
  reset_indent
103
101
  push :block_line
104
102
  end
105
103
 
106
104
  # indentation spaces
107
- rule /[ ]*(?!\s|$)/ do
108
- text, err = save_indent
109
- token 'Text', text
110
- token 'Error', err
105
+ rule /[ ]*(?!\s|$)/ do |m|
106
+ text, err = save_indent(m[0])
107
+ token Text, text
108
+ token Error, err
111
109
  push :block_line; push :indentation
112
110
  end
113
111
  end
114
112
 
115
113
  state :indentation do
116
- rule(/\s*?\n/) { token 'Text'; pop! 2 }
114
+ rule(/\s*?\n/) { token Text; pop! 2 }
117
115
  # whitespace preceding block collection indicators
118
- rule /[ ]+(?=[-:?](?:[ ]|$))/ do
119
- token 'Text'
120
- continue_indent
116
+ rule /[ ]+(?=[-:?](?:[ ]|$))/ do |m|
117
+ token Text
118
+ continue_indent(m[0])
121
119
  end
122
120
 
123
121
  # block collection indicators
124
- rule(/[?:-](?=[ ]|$)/) { token 'Punctuation.Indicator'; set_indent }
122
+ rule(/[?:-](?=[ ]|$)/) { token Punctuation::Indicator; set_indent }
125
123
 
126
124
  # the beginning of a block line
127
- rule(/[ ]*/) { token 'Text'; continue_indent; pop! }
125
+ rule(/[ ]*/) { |m| token Text; continue_indent(m[0]); pop! }
128
126
  end
129
127
 
130
128
  # indented line in the block context
131
129
  state :block_line do
132
130
  # line end
133
- rule /[ ]*(?=#|$)/, 'Text', :pop!
134
- rule /[ ]+/, 'Text'
131
+ rule /[ ]*(?=#|$)/, Text, :pop!
132
+ rule /[ ]+/, Text
135
133
  # tags, anchors, and aliases
136
134
  mixin :descriptors
137
135
  # block collections and scalars
@@ -141,54 +139,54 @@ module Rouge
141
139
 
142
140
  # a plain scalar
143
141
  rule /(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
144
- token 'Name.Variable'
142
+ token Name::Variable
145
143
  push :plain_scalar_in_block_context
146
144
  end
147
145
  end
148
146
 
149
147
  state :descriptors do
150
148
  # a full-form tag
151
- rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, 'Keyword.Type'
149
+ rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
152
150
 
153
151
  # a tag in the form '!', '!suffix' or '!handle!suffix'
154
152
  rule %r(
155
153
  (?:![\w-]+)? # handle
156
154
  !(?:[\w;/?:@&=+$,.!~*\'()\[\]%-]*) # suffix
157
- )x, 'Keyword.Type'
155
+ )x, Keyword::Type
158
156
 
159
157
  # an anchor
160
- rule /&[\w-]+/, 'Name.Label'
158
+ rule /&[\w-]+/, Name::Label
161
159
 
162
160
  # an alias
163
- rule /\*[\w-]+/, 'Name.Variable'
161
+ rule /\*[\w-]+/, Name::Variable
164
162
  end
165
163
 
166
164
  state :block_nodes do
167
165
  # implicit key
168
166
  rule /:(?=\s|$)/ do
169
- token 'Punctuation.Indicator'
167
+ token Punctuation::Indicator
170
168
  set_indent :implicit => true
171
169
  end
172
170
 
173
171
  # literal and folded scalars
174
172
  rule /[\|>]/ do
175
- token 'Punctuation.Indicator'
173
+ token Punctuation::Indicator
176
174
  push :block_scalar_content
177
175
  push :block_scalar_header
178
176
  end
179
177
  end
180
178
 
181
179
  state :flow_nodes do
182
- rule /\[/, 'Punctuation.Indicator', :flow_sequence
183
- rule /\{/, 'Punctuation.Indicator', :flow_mapping
184
- rule /'/, 'Literal.String.Single', :single_quoted_scalar
185
- rule /"/, 'Literal.String.Double', :double_quoted_scalar
180
+ rule /\[/, Punctuation::Indicator, :flow_sequence
181
+ rule /\{/, Punctuation::Indicator, :flow_mapping
182
+ rule /'/, Str::Single, :single_quoted_scalar
183
+ rule /"/, Str::Double, :double_quoted_scalar
186
184
  end
187
185
 
188
186
  state :flow_collection do
189
- rule /\s+/m, 'Text'
187
+ rule /\s+/m, Text
190
188
  mixin :basic
191
- rule /[?:,]/, 'Punctuation.Indicator'
189
+ rule /[?:,]/, Punctuation::Indicator
192
190
  mixin :descriptors
193
191
  mixin :flow_nodes
194
192
 
@@ -198,17 +196,17 @@ module Rouge
198
196
  end
199
197
 
200
198
  state :flow_sequence do
201
- rule /\]/, 'Punctuation.Indicator', :pop!
199
+ rule /\]/, Punctuation::Indicator, :pop!
202
200
  mixin :flow_collection
203
201
  end
204
202
 
205
203
  state :flow_mapping do
206
- rule /\}/, 'Punctuation.Indicator', :pop!
204
+ rule /\}/, Punctuation::Indicator, :pop!
207
205
  mixin :flow_collection
208
206
  end
209
207
 
210
208
  state :block_scalar_content do
211
- rule /\n+/, 'Text'
209
+ rule /\n+/, Text
212
210
 
213
211
  # empty lines never dedent, but they might be part of the scalar.
214
212
  rule /^[ ]+$/ do |m|
@@ -217,14 +215,14 @@ module Rouge
217
215
 
218
216
  indent_mark = @block_scalar_indent || indent_size
219
217
 
220
- token 'Text', text[0...indent_mark]
221
- token 'Name.Constant', text[indent_mark..-1]
218
+ token Text, text[0...indent_mark]
219
+ token Name::Constant, text[indent_mark..-1]
222
220
  end
223
221
 
224
222
  # TODO: ^ doesn't actually seem to affect the match at all.
225
223
  # Find a way to work around this limitation.
226
224
  rule /^[ ]*/ do |m|
227
- token 'Text'
225
+ token Text
228
226
 
229
227
  indent_size = m[0].size
230
228
 
@@ -236,7 +234,7 @@ module Rouge
236
234
  end
237
235
  end
238
236
 
239
- rule /[^\n\r\f\v]+/, 'Name.Constant'
237
+ rule /[^\n\r\f\v]+/, Name::Constant
240
238
  end
241
239
 
242
240
  state :block_scalar_header do
@@ -247,7 +245,7 @@ module Rouge
247
245
  )(?=[ ]|$)
248
246
  )x do |m|
249
247
  @block_scalar_indent = nil
250
- pop!; push :ignored_line
248
+ goto :ignored_line
251
249
  next if m[0].empty?
252
250
 
253
251
  increment = m[1] || m[2]
@@ -255,62 +253,61 @@ module Rouge
255
253
  @block_scalar_indent = indent + increment.to_i
256
254
  end
257
255
 
258
- token 'Punctuation.Indicator'
256
+ token Punctuation::Indicator
259
257
  end
260
258
  end
261
259
 
262
260
  state :ignored_line do
263
261
  mixin :basic
264
- rule /[ ]+/, 'Text'
265
- rule /\n/, 'Text', :pop!
262
+ rule /[ ]+/, Text
263
+ rule /\n/, Text, :pop!
266
264
  end
267
265
 
268
266
  state :quoted_scalar_whitespaces do
269
267
  # leading and trailing whitespace is ignored
270
- rule /^[ ]+/, 'Text'
271
- rule /[ ]+$/, 'Text'
268
+ rule /^[ ]+/, Text
269
+ rule /[ ]+$/, Text
272
270
 
273
- rule /\n+/m, 'Text'
271
+ rule /\n+/m, Text
274
272
 
275
- rule /[ ]+/, 'Name.Variable'
273
+ rule /[ ]+/, Name::Variable
276
274
  end
277
275
 
278
276
  state :single_quoted_scalar do
279
277
  mixin :quoted_scalar_whitespaces
280
- rule /\\'/, 'Literal.String.Escape'
281
- rule /'/, 'Literal.String', :pop!
282
- rule /[^\s']+/, 'Literal.String'
278
+ rule /\\'/, Str::Escape
279
+ rule /'/, Str, :pop!
280
+ rule /[^\s']+/, Str
283
281
  end
284
282
 
285
283
  state :double_quoted_scalar do
286
- rule /"/, 'Literal.String', :pop!
284
+ rule /"/, Str, :pop!
287
285
  mixin :quoted_scalar_whitespaces
288
286
  # escapes
289
- rule /\\[0abt\tn\nvfre "\\N_LP]/, 'Literal.String.Escape'
287
+ rule /\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
290
288
  rule /\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
291
- 'Literal.String.Escape'
292
- rule /[^ \t\n\r\f\v"\\]+/, 'Literal.String'
289
+ Str::Escape
290
+ rule /[^ \t\n\r\f\v"\\]+/, Str
293
291
  end
294
292
 
295
293
  state :plain_scalar_in_block_context_new_line do
296
- rule /^[ ]+\n/, 'Text'
297
- rule /\n+/m, 'Text'
294
+ rule /^[ ]+\n/, Text
295
+ rule /\n+/m, Text
298
296
  rule /^(?=---|\.\.\.)/ do
299
297
  pop! 3
300
298
  end
301
299
 
302
300
  # dedent detection
303
301
  rule /^[ ]*/ do |m|
304
- token 'Text'
302
+ token Text
305
303
  pop!
306
304
 
307
305
  indent_size = m[0].size
308
306
 
309
307
  # dedent = end of scalar
310
- if m[0].size <= self.indent
308
+ if indent_size <= self.indent
311
309
  pop!
312
- val, err = save_indent
313
- # push :block_line
310
+ save_indent(m[0])
314
311
  push :indentation
315
312
  end
316
313
  end
@@ -318,34 +315,34 @@ module Rouge
318
315
 
319
316
  state :plain_scalar_in_block_context do
320
317
  # the : indicator ends a scalar
321
- rule /[ ]*(?=:[ \n]|:$)/, 'Text', :pop!
322
- rule /[ ]*:/, 'Literal.String'
323
- rule /[ ]+(?=#)/, 'Text', :pop!
324
- rule /[ ]+$/, 'Text'
318
+ rule /[ ]*(?=:[ \n]|:$)/, Text, :pop!
319
+ rule /[ ]*:/, Str
320
+ rule /[ ]+(?=#)/, Text, :pop!
321
+ rule /[ ]+$/, Text
325
322
  # check for new documents or dedents at the new line
326
323
  rule /\n+/ do
327
- token 'Text'
324
+ token Text
328
325
  push :plain_scalar_in_block_context_new_line
329
326
  end
330
327
 
331
- rule /[ ]+/, 'Literal.String'
328
+ rule /[ ]+/, Str
332
329
  # regular non-whitespace characters
333
- rule /[^\s:]+/, 'Literal.String'
330
+ rule /[^\s:]+/, Str
334
331
  end
335
332
 
336
333
  state :plain_scalar_in_flow_context do
337
- rule /[ ]*(?=[,:?\[\]{}])/, 'Text', :pop!
338
- rule /[ ]+(?=#)/, 'Text', :pop!
339
- rule /^[ ]+/, 'Text'
340
- rule /[ ]+$/, 'Text'
341
- rule /\n+/, 'Text'
342
- rule /[ ]+/, 'Name.Variable'
343
- rule /[^\s,:?\[\]{}]+/, 'Name.Variable'
334
+ rule /[ ]*(?=[,:?\[\]{}])/, Text, :pop!
335
+ rule /[ ]+(?=#)/, Text, :pop!
336
+ rule /^[ ]+/, Text
337
+ rule /[ ]+$/, Text
338
+ rule /\n+/, Text
339
+ rule /[ ]+/, Name::Variable
340
+ rule /[^\s,:?\[\]{}]+/, Name::Variable
344
341
  end
345
342
 
346
343
  state :yaml_directive do
347
344
  rule /([ ]+)(\d+\.\d+)/ do
348
- group 'Text'; group 'Number'
345
+ group Text; group Num
349
346
  pop!; push :ignored_line
350
347
  end
351
348
  end
@@ -355,8 +352,8 @@ module Rouge
355
352
  ([ ]+)(!|![\w-]*!) # prefix
356
353
  ([ ]+)(!|!?[\w;/?:@&=+$,.!~*'()\[\]%-]+) # tag handle
357
354
  )x do
358
- group 'Text'; group 'Keyword.Type'
359
- group 'Text'; group 'Keyword.Type'
355
+ group Text; group Keyword::Type
356
+ group Text; group Keyword::Type
360
357
  pop!; push :ignored_line
361
358
  end
362
359
  end
@@ -6,7 +6,7 @@ module Rouge
6
6
  module Plugins
7
7
  module Redcarpet
8
8
  def block_code(code, language)
9
- lexer = Lexer.find_fancy(language, code) || Lexers::Text
9
+ lexer = Lexer.find_fancy(language, code) || Lexers::PlainText
10
10
 
11
11
  # XXX HACK: Redcarpet strips hard tabs out of code blocks,
12
12
  # so we assume you're not using leading spaces that aren't tabs,
@@ -165,6 +165,7 @@ module Rouge
165
165
  # start_procs.
166
166
  def reset!
167
167
  @stack = nil
168
+ @current_stream = nil
168
169
 
169
170
  self.class.start_procs.each do |pr|
170
171
  instance_eval(&pr)
@@ -186,6 +187,8 @@ module Rouge
186
187
  def stream_tokens(str, &b)
187
188
  stream = StringScanner.new(str)
188
189
 
190
+ @current_stream = stream
191
+
189
192
  until stream.eos?
190
193
  debug { "lexer: #{self.class.tag}" }
191
194
  debug { "stack: #{stack.map(&:name).inspect}" }
@@ -194,7 +197,7 @@ module Rouge
194
197
 
195
198
  if !success
196
199
  debug { " no match, yielding Error" }
197
- b.call(Token['Error'], stream.getch)
200
+ b.call(Token::Tokens::Error, stream.getch)
198
201
  end
199
202
  end
200
203
  end
@@ -231,9 +234,7 @@ module Rouge
231
234
  def run_callback(stream, callback, &output_stream)
232
235
  with_output_stream(output_stream) do
233
236
  @group_count = 0
234
- @last_match = stream
235
237
  instance_exec(stream, &callback)
236
- @last_match = nil
237
238
  end
238
239
  end
239
240
 
@@ -274,18 +275,20 @@ module Rouge
274
275
  # (optional) the string value to yield. If absent, this defaults
275
276
  # to the entire last match.
276
277
  def token(tok, val=:__absent__)
277
- val = @last_match[0] if val == :__absent__
278
- val ||= ''
279
-
280
- raise 'no output stream' unless @output_stream
281
-
282
- @output_stream << [Token[tok], val] unless val.empty?
278
+ val = @current_stream[0] if val == :__absent__
279
+ yield_token(tok, val)
283
280
  end
284
281
 
285
282
  # Yield a token with the next matched group. Subsequent calls
286
283
  # to this method will yield subsequent groups.
287
284
  def group(tok)
288
- token(tok, @last_match[@group_count += 1])
285
+ yield_token(tok, @current_stream[@group_count += 1])
286
+ end
287
+
288
+ def groups(*tokens)
289
+ tokens.each_with_index do |tok, i|
290
+ yield_token(tok, @current_stream[i+1])
291
+ end
289
292
  end
290
293
 
291
294
  # Delegate the lex to another lexer. The #lex method will be called
@@ -299,7 +302,7 @@ module Rouge
299
302
  # The text to delegate. This defaults to the last matched string.
300
303
  def delegate(lexer, text=nil)
301
304
  debug { " delegating to #{lexer.inspect}" }
302
- text ||= @last_match[0]
305
+ text ||= @current_stream[0]
303
306
 
304
307
  lexer.lex(text, :continue => true) do |tok, val|
305
308
  debug { " delegated token: #{tok.inspect}, #{val.inspect}" }
@@ -336,6 +339,12 @@ module Rouge
336
339
  nil
337
340
  end
338
341
 
342
+ # replace the head of the stack with the given state
343
+ def goto(state_name)
344
+ raise 'empty stack!' if stack.empty?
345
+ stack[-1] = get_state(state_name)
346
+ end
347
+
339
348
  # reset the stack back to `[:root]`.
340
349
  def reset_stack
341
350
  debug { ' resetting stack' }
@@ -345,7 +354,10 @@ module Rouge
345
354
 
346
355
  # Check if `state_name` is in the state stack.
347
356
  def in_state?(state_name)
348
- stack.map(&:name).include? state_name.to_s
357
+ state_name = state_name.to_s
358
+ stack.any? do |state|
359
+ state.name == state_name.to_s
360
+ end
349
361
  end
350
362
 
351
363
  # Check if `state_name` is the state on top of the state stack.
@@ -357,8 +369,8 @@ module Rouge
357
369
  def with_output_stream(output_stream, &b)
358
370
  old_output_stream = @output_stream
359
371
  @output_stream = Enumerator::Yielder.new do |tok, val|
360
- debug { " yielding #{tok.to_s.inspect}, #{val.inspect}" }
361
- output_stream.call(Token[tok], val)
372
+ debug { " yielding #{tok.qualname}, #{val.inspect}" }
373
+ output_stream.call(tok, val)
362
374
  end
363
375
 
364
376
  yield
@@ -366,5 +378,10 @@ module Rouge
366
378
  ensure
367
379
  @output_stream = old_output_stream
368
380
  end
381
+
382
+ def yield_token(tok, val)
383
+ return if val.nil? || val.empty?
384
+ @output_stream.yield(tok, val)
385
+ end
369
386
  end
370
387
  end