rouge 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +2 -0
- data/bin/rougify +1 -1
- data/lib/rouge/demos/{text → plaintext} +0 -0
- data/lib/rouge/formatters/html.rb +13 -8
- data/lib/rouge/formatters/terminal256.rb +15 -11
- data/lib/rouge/lexer.rb +3 -1
- data/lib/rouge/lexers/c.rb +65 -55
- data/lib/rouge/lexers/clojure.rb +19 -19
- data/lib/rouge/lexers/coffeescript.rb +42 -42
- data/lib/rouge/lexers/common_lisp.rb +49 -48
- data/lib/rouge/lexers/conf.rb +6 -6
- data/lib/rouge/lexers/cpp.rb +48 -48
- data/lib/rouge/lexers/csharp.rb +25 -25
- data/lib/rouge/lexers/css.rb +32 -33
- data/lib/rouge/lexers/diff.rb +8 -8
- data/lib/rouge/lexers/elixir.rb +30 -30
- data/lib/rouge/lexers/erb.rb +5 -5
- data/lib/rouge/lexers/erlang.rb +38 -38
- data/lib/rouge/lexers/factor.rb +56 -56
- data/lib/rouge/lexers/gherkin.rb +27 -27
- data/lib/rouge/lexers/go.rb +21 -21
- data/lib/rouge/lexers/groovy.rb +32 -32
- data/lib/rouge/lexers/haml.rb +35 -35
- data/lib/rouge/lexers/handlebars.rb +27 -27
- data/lib/rouge/lexers/haskell.rb +78 -79
- data/lib/rouge/lexers/html.rb +28 -28
- data/lib/rouge/lexers/http.rb +13 -13
- data/lib/rouge/lexers/ini.rb +13 -13
- data/lib/rouge/lexers/io.rb +19 -19
- data/lib/rouge/lexers/java.rb +29 -29
- data/lib/rouge/lexers/javascript.rb +49 -52
- data/lib/rouge/lexers/literate_haskell.rb +5 -5
- data/lib/rouge/lexers/llvm.rb +14 -14
- data/lib/rouge/lexers/lua.rb +33 -33
- data/lib/rouge/lexers/make.rb +26 -26
- data/lib/rouge/lexers/markdown.rb +49 -49
- data/lib/rouge/lexers/nginx.rb +19 -19
- data/lib/rouge/lexers/perl.rb +53 -54
- data/lib/rouge/lexers/php.rb +42 -43
- data/lib/rouge/lexers/{text.rb → plain_text.rb} +3 -2
- data/lib/rouge/lexers/prolog.rb +19 -19
- data/lib/rouge/lexers/puppet.rb +40 -40
- data/lib/rouge/lexers/python.rb +63 -63
- data/lib/rouge/lexers/r.rb +15 -15
- data/lib/rouge/lexers/racket.rb +29 -29
- data/lib/rouge/lexers/ruby.rb +94 -97
- data/lib/rouge/lexers/rust.rb +37 -37
- data/lib/rouge/lexers/sass.rb +15 -15
- data/lib/rouge/lexers/sass/common.rb +144 -136
- data/lib/rouge/lexers/scheme.rb +25 -25
- data/lib/rouge/lexers/scss.rb +7 -7
- data/lib/rouge/lexers/sed.rb +39 -39
- data/lib/rouge/lexers/shell.rb +45 -45
- data/lib/rouge/lexers/smalltalk.rb +36 -36
- data/lib/rouge/lexers/sql.rb +27 -27
- data/lib/rouge/lexers/tcl.rb +34 -34
- data/lib/rouge/lexers/tex.rb +24 -24
- data/lib/rouge/lexers/toml.rb +18 -18
- data/lib/rouge/lexers/viml.rb +19 -19
- data/lib/rouge/lexers/xml.rb +16 -16
- data/lib/rouge/lexers/yaml.rb +78 -81
- data/lib/rouge/plugins/redcarpet.rb +1 -1
- data/lib/rouge/regex_lexer.rb +31 -14
- data/lib/rouge/template_lexer.rb +1 -1
- data/lib/rouge/theme.rb +12 -10
- data/lib/rouge/themes/thankful_eyes.rb +43 -43
- data/lib/rouge/token.rb +145 -148
- data/lib/rouge/version.rb +1 -1
- metadata +4 -4
data/lib/rouge/lexers/xml.rb
CHANGED
@@ -21,35 +21,35 @@ module Rouge
|
|
21
21
|
end
|
22
22
|
|
23
23
|
state :root do
|
24
|
-
rule /[^<&]+/,
|
25
|
-
rule /&\S*?;/,
|
26
|
-
rule /<!\[CDATA\[.*?\]\]\>/,
|
27
|
-
rule /<!--/,
|
28
|
-
rule /<\?.*?\?>/,
|
29
|
-
rule /<![^>]*>/,
|
24
|
+
rule /[^<&]+/, Text
|
25
|
+
rule /&\S*?;/, Name::Entity
|
26
|
+
rule /<!\[CDATA\[.*?\]\]\>/, Comment::Preproc
|
27
|
+
rule /<!--/, Comment, :comment
|
28
|
+
rule /<\?.*?\?>/, Comment::Preproc
|
29
|
+
rule /<![^>]*>/, Comment::Preproc
|
30
30
|
|
31
31
|
# open tags
|
32
|
-
rule %r(<\s*[\w:.-]+)m,
|
32
|
+
rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
|
33
33
|
|
34
34
|
# self-closing tags
|
35
|
-
rule %r(<\s*/\s*[\w:.-]+\s*>)m,
|
35
|
+
rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
|
36
36
|
end
|
37
37
|
|
38
38
|
state :comment do
|
39
|
-
rule /[^-]+/m,
|
40
|
-
rule /-->/,
|
41
|
-
rule /-/,
|
39
|
+
rule /[^-]+/m, Comment
|
40
|
+
rule /-->/, Comment, :pop!
|
41
|
+
rule /-/, Comment
|
42
42
|
end
|
43
43
|
|
44
44
|
state :tag do
|
45
|
-
rule /\s+/m,
|
46
|
-
rule /[\w.:-]+\s*=/m,
|
47
|
-
rule %r(/?\s*>),
|
45
|
+
rule /\s+/m, Text
|
46
|
+
rule /[\w.:-]+\s*=/m, Name::Attribute, :attr
|
47
|
+
rule %r(/?\s*>), Name::Tag, :pop!
|
48
48
|
end
|
49
49
|
|
50
50
|
state :attr do
|
51
|
-
rule /\s+/m,
|
52
|
-
rule /".*?"|'.*?'|[^\s>]+/,
|
51
|
+
rule /\s+/m, Text
|
52
|
+
rule /".*?"|'.*?'|[^\s>]+/, Str, :pop!
|
53
53
|
end
|
54
54
|
end
|
55
55
|
end
|
data/lib/rouge/lexers/yaml.rb
CHANGED
@@ -37,9 +37,7 @@ module Rouge
|
|
37
37
|
end
|
38
38
|
|
39
39
|
# Save a possible indentation level
|
40
|
-
def save_indent(
|
41
|
-
debug { " yaml: save_indent" }
|
42
|
-
match = @last_match[0]
|
40
|
+
def save_indent(match)
|
43
41
|
@next_indent = match.size
|
44
42
|
debug { " yaml: indent: #{self.indent}/#@next_indent" }
|
45
43
|
debug { " yaml: popping indent stack - before: #@indent_stack" }
|
@@ -55,9 +53,9 @@ module Rouge
|
|
55
53
|
end
|
56
54
|
end
|
57
55
|
|
58
|
-
def continue_indent
|
56
|
+
def continue_indent(match)
|
59
57
|
debug { " yaml: continue_indent" }
|
60
|
-
@next_indent +=
|
58
|
+
@next_indent += match.size
|
61
59
|
end
|
62
60
|
|
63
61
|
def set_indent(opts={})
|
@@ -73,65 +71,65 @@ module Rouge
|
|
73
71
|
start { reset_indent }
|
74
72
|
|
75
73
|
state :basic do
|
76
|
-
rule /#.*$/,
|
74
|
+
rule /#.*$/, Comment::Single
|
77
75
|
end
|
78
76
|
|
79
77
|
state :root do
|
80
78
|
mixin :basic
|
81
79
|
|
82
|
-
rule /\n+/,
|
80
|
+
rule /\n+/, Text
|
83
81
|
|
84
82
|
# trailing or pre-comment whitespace
|
85
|
-
rule /[ ]+(?=#|$)/,
|
83
|
+
rule /[ ]+(?=#|$)/, Text
|
86
84
|
|
87
85
|
rule /^%YAML\b/ do
|
88
|
-
token
|
86
|
+
token Name::Tag
|
89
87
|
reset_indent
|
90
88
|
push :yaml_directive
|
91
89
|
end
|
92
90
|
|
93
91
|
rule /^%TAG\b/ do
|
94
|
-
token
|
92
|
+
token Name::Tag
|
95
93
|
reset_indent
|
96
94
|
push :tag_directive
|
97
95
|
end
|
98
96
|
|
99
97
|
# doc-start and doc-end indicators
|
100
98
|
rule /^(?:---|\.\.\.)(?= |$)/ do
|
101
|
-
token
|
99
|
+
token Name::Namespace
|
102
100
|
reset_indent
|
103
101
|
push :block_line
|
104
102
|
end
|
105
103
|
|
106
104
|
# indentation spaces
|
107
|
-
rule /[ ]*(?!\s|$)/ do
|
108
|
-
text, err = save_indent
|
109
|
-
token
|
110
|
-
token
|
105
|
+
rule /[ ]*(?!\s|$)/ do |m|
|
106
|
+
text, err = save_indent(m[0])
|
107
|
+
token Text, text
|
108
|
+
token Error, err
|
111
109
|
push :block_line; push :indentation
|
112
110
|
end
|
113
111
|
end
|
114
112
|
|
115
113
|
state :indentation do
|
116
|
-
rule(/\s*?\n/) { token
|
114
|
+
rule(/\s*?\n/) { token Text; pop! 2 }
|
117
115
|
# whitespace preceding block collection indicators
|
118
|
-
rule /[ ]+(?=[-:?](?:[ ]|$))/ do
|
119
|
-
token
|
120
|
-
continue_indent
|
116
|
+
rule /[ ]+(?=[-:?](?:[ ]|$))/ do |m|
|
117
|
+
token Text
|
118
|
+
continue_indent(m[0])
|
121
119
|
end
|
122
120
|
|
123
121
|
# block collection indicators
|
124
|
-
rule(/[?:-](?=[ ]|$)/) { token
|
122
|
+
rule(/[?:-](?=[ ]|$)/) { token Punctuation::Indicator; set_indent }
|
125
123
|
|
126
124
|
# the beginning of a block line
|
127
|
-
rule(/[ ]*/) { token
|
125
|
+
rule(/[ ]*/) { |m| token Text; continue_indent(m[0]); pop! }
|
128
126
|
end
|
129
127
|
|
130
128
|
# indented line in the block context
|
131
129
|
state :block_line do
|
132
130
|
# line end
|
133
|
-
rule /[ ]*(?=#|$)/,
|
134
|
-
rule /[ ]+/,
|
131
|
+
rule /[ ]*(?=#|$)/, Text, :pop!
|
132
|
+
rule /[ ]+/, Text
|
135
133
|
# tags, anchors, and aliases
|
136
134
|
mixin :descriptors
|
137
135
|
# block collections and scalars
|
@@ -141,54 +139,54 @@ module Rouge
|
|
141
139
|
|
142
140
|
# a plain scalar
|
143
141
|
rule /(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
|
144
|
-
token
|
142
|
+
token Name::Variable
|
145
143
|
push :plain_scalar_in_block_context
|
146
144
|
end
|
147
145
|
end
|
148
146
|
|
149
147
|
state :descriptors do
|
150
148
|
# a full-form tag
|
151
|
-
rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/,
|
149
|
+
rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
|
152
150
|
|
153
151
|
# a tag in the form '!', '!suffix' or '!handle!suffix'
|
154
152
|
rule %r(
|
155
153
|
(?:![\w-]+)? # handle
|
156
154
|
!(?:[\w;/?:@&=+$,.!~*\'()\[\]%-]*) # suffix
|
157
|
-
)x,
|
155
|
+
)x, Keyword::Type
|
158
156
|
|
159
157
|
# an anchor
|
160
|
-
rule /&[\w-]+/,
|
158
|
+
rule /&[\w-]+/, Name::Label
|
161
159
|
|
162
160
|
# an alias
|
163
|
-
rule /\*[\w-]+/,
|
161
|
+
rule /\*[\w-]+/, Name::Variable
|
164
162
|
end
|
165
163
|
|
166
164
|
state :block_nodes do
|
167
165
|
# implicit key
|
168
166
|
rule /:(?=\s|$)/ do
|
169
|
-
token
|
167
|
+
token Punctuation::Indicator
|
170
168
|
set_indent :implicit => true
|
171
169
|
end
|
172
170
|
|
173
171
|
# literal and folded scalars
|
174
172
|
rule /[\|>]/ do
|
175
|
-
token
|
173
|
+
token Punctuation::Indicator
|
176
174
|
push :block_scalar_content
|
177
175
|
push :block_scalar_header
|
178
176
|
end
|
179
177
|
end
|
180
178
|
|
181
179
|
state :flow_nodes do
|
182
|
-
rule /\[/,
|
183
|
-
rule /\{/,
|
184
|
-
rule /'/,
|
185
|
-
rule /"/,
|
180
|
+
rule /\[/, Punctuation::Indicator, :flow_sequence
|
181
|
+
rule /\{/, Punctuation::Indicator, :flow_mapping
|
182
|
+
rule /'/, Str::Single, :single_quoted_scalar
|
183
|
+
rule /"/, Str::Double, :double_quoted_scalar
|
186
184
|
end
|
187
185
|
|
188
186
|
state :flow_collection do
|
189
|
-
rule /\s+/m,
|
187
|
+
rule /\s+/m, Text
|
190
188
|
mixin :basic
|
191
|
-
rule /[?:,]/,
|
189
|
+
rule /[?:,]/, Punctuation::Indicator
|
192
190
|
mixin :descriptors
|
193
191
|
mixin :flow_nodes
|
194
192
|
|
@@ -198,17 +196,17 @@ module Rouge
|
|
198
196
|
end
|
199
197
|
|
200
198
|
state :flow_sequence do
|
201
|
-
rule /\]/,
|
199
|
+
rule /\]/, Punctuation::Indicator, :pop!
|
202
200
|
mixin :flow_collection
|
203
201
|
end
|
204
202
|
|
205
203
|
state :flow_mapping do
|
206
|
-
rule /\}/,
|
204
|
+
rule /\}/, Punctuation::Indicator, :pop!
|
207
205
|
mixin :flow_collection
|
208
206
|
end
|
209
207
|
|
210
208
|
state :block_scalar_content do
|
211
|
-
rule /\n+/,
|
209
|
+
rule /\n+/, Text
|
212
210
|
|
213
211
|
# empty lines never dedent, but they might be part of the scalar.
|
214
212
|
rule /^[ ]+$/ do |m|
|
@@ -217,14 +215,14 @@ module Rouge
|
|
217
215
|
|
218
216
|
indent_mark = @block_scalar_indent || indent_size
|
219
217
|
|
220
|
-
token
|
221
|
-
token
|
218
|
+
token Text, text[0...indent_mark]
|
219
|
+
token Name::Constant, text[indent_mark..-1]
|
222
220
|
end
|
223
221
|
|
224
222
|
# TODO: ^ doesn't actually seem to affect the match at all.
|
225
223
|
# Find a way to work around this limitation.
|
226
224
|
rule /^[ ]*/ do |m|
|
227
|
-
token
|
225
|
+
token Text
|
228
226
|
|
229
227
|
indent_size = m[0].size
|
230
228
|
|
@@ -236,7 +234,7 @@ module Rouge
|
|
236
234
|
end
|
237
235
|
end
|
238
236
|
|
239
|
-
rule /[^\n\r\f\v]+/,
|
237
|
+
rule /[^\n\r\f\v]+/, Name::Constant
|
240
238
|
end
|
241
239
|
|
242
240
|
state :block_scalar_header do
|
@@ -247,7 +245,7 @@ module Rouge
|
|
247
245
|
)(?=[ ]|$)
|
248
246
|
)x do |m|
|
249
247
|
@block_scalar_indent = nil
|
250
|
-
|
248
|
+
goto :ignored_line
|
251
249
|
next if m[0].empty?
|
252
250
|
|
253
251
|
increment = m[1] || m[2]
|
@@ -255,62 +253,61 @@ module Rouge
|
|
255
253
|
@block_scalar_indent = indent + increment.to_i
|
256
254
|
end
|
257
255
|
|
258
|
-
token
|
256
|
+
token Punctuation::Indicator
|
259
257
|
end
|
260
258
|
end
|
261
259
|
|
262
260
|
state :ignored_line do
|
263
261
|
mixin :basic
|
264
|
-
rule /[ ]+/,
|
265
|
-
rule /\n/,
|
262
|
+
rule /[ ]+/, Text
|
263
|
+
rule /\n/, Text, :pop!
|
266
264
|
end
|
267
265
|
|
268
266
|
state :quoted_scalar_whitespaces do
|
269
267
|
# leading and trailing whitespace is ignored
|
270
|
-
rule /^[ ]+/,
|
271
|
-
rule /[ ]+$/,
|
268
|
+
rule /^[ ]+/, Text
|
269
|
+
rule /[ ]+$/, Text
|
272
270
|
|
273
|
-
rule /\n+/m,
|
271
|
+
rule /\n+/m, Text
|
274
272
|
|
275
|
-
rule /[ ]+/,
|
273
|
+
rule /[ ]+/, Name::Variable
|
276
274
|
end
|
277
275
|
|
278
276
|
state :single_quoted_scalar do
|
279
277
|
mixin :quoted_scalar_whitespaces
|
280
|
-
rule /\\'/,
|
281
|
-
rule /'/,
|
282
|
-
rule /[^\s']+/,
|
278
|
+
rule /\\'/, Str::Escape
|
279
|
+
rule /'/, Str, :pop!
|
280
|
+
rule /[^\s']+/, Str
|
283
281
|
end
|
284
282
|
|
285
283
|
state :double_quoted_scalar do
|
286
|
-
rule /"/,
|
284
|
+
rule /"/, Str, :pop!
|
287
285
|
mixin :quoted_scalar_whitespaces
|
288
286
|
# escapes
|
289
|
-
rule /\\[0abt\tn\nvfre "\\N_LP]/,
|
287
|
+
rule /\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
|
290
288
|
rule /\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
|
291
|
-
|
292
|
-
rule /[^ \t\n\r\f\v"\\]+/,
|
289
|
+
Str::Escape
|
290
|
+
rule /[^ \t\n\r\f\v"\\]+/, Str
|
293
291
|
end
|
294
292
|
|
295
293
|
state :plain_scalar_in_block_context_new_line do
|
296
|
-
rule /^[ ]+\n/,
|
297
|
-
rule /\n+/m,
|
294
|
+
rule /^[ ]+\n/, Text
|
295
|
+
rule /\n+/m, Text
|
298
296
|
rule /^(?=---|\.\.\.)/ do
|
299
297
|
pop! 3
|
300
298
|
end
|
301
299
|
|
302
300
|
# dedent detection
|
303
301
|
rule /^[ ]*/ do |m|
|
304
|
-
token
|
302
|
+
token Text
|
305
303
|
pop!
|
306
304
|
|
307
305
|
indent_size = m[0].size
|
308
306
|
|
309
307
|
# dedent = end of scalar
|
310
|
-
if
|
308
|
+
if indent_size <= self.indent
|
311
309
|
pop!
|
312
|
-
|
313
|
-
# push :block_line
|
310
|
+
save_indent(m[0])
|
314
311
|
push :indentation
|
315
312
|
end
|
316
313
|
end
|
@@ -318,34 +315,34 @@ module Rouge
|
|
318
315
|
|
319
316
|
state :plain_scalar_in_block_context do
|
320
317
|
# the : indicator ends a scalar
|
321
|
-
rule /[ ]*(?=:[ \n]|:$)/,
|
322
|
-
rule /[ ]*:/,
|
323
|
-
rule /[ ]+(?=#)/,
|
324
|
-
rule /[ ]+$/,
|
318
|
+
rule /[ ]*(?=:[ \n]|:$)/, Text, :pop!
|
319
|
+
rule /[ ]*:/, Str
|
320
|
+
rule /[ ]+(?=#)/, Text, :pop!
|
321
|
+
rule /[ ]+$/, Text
|
325
322
|
# check for new documents or dedents at the new line
|
326
323
|
rule /\n+/ do
|
327
|
-
token
|
324
|
+
token Text
|
328
325
|
push :plain_scalar_in_block_context_new_line
|
329
326
|
end
|
330
327
|
|
331
|
-
rule /[ ]+/,
|
328
|
+
rule /[ ]+/, Str
|
332
329
|
# regular non-whitespace characters
|
333
|
-
rule /[^\s:]+/,
|
330
|
+
rule /[^\s:]+/, Str
|
334
331
|
end
|
335
332
|
|
336
333
|
state :plain_scalar_in_flow_context do
|
337
|
-
rule /[ ]*(?=[,:?\[\]{}])/,
|
338
|
-
rule /[ ]+(?=#)/,
|
339
|
-
rule /^[ ]+/,
|
340
|
-
rule /[ ]+$/,
|
341
|
-
rule /\n+/,
|
342
|
-
rule /[ ]+/,
|
343
|
-
rule /[^\s,:?\[\]{}]+/,
|
334
|
+
rule /[ ]*(?=[,:?\[\]{}])/, Text, :pop!
|
335
|
+
rule /[ ]+(?=#)/, Text, :pop!
|
336
|
+
rule /^[ ]+/, Text
|
337
|
+
rule /[ ]+$/, Text
|
338
|
+
rule /\n+/, Text
|
339
|
+
rule /[ ]+/, Name::Variable
|
340
|
+
rule /[^\s,:?\[\]{}]+/, Name::Variable
|
344
341
|
end
|
345
342
|
|
346
343
|
state :yaml_directive do
|
347
344
|
rule /([ ]+)(\d+\.\d+)/ do
|
348
|
-
group
|
345
|
+
group Text; group Num
|
349
346
|
pop!; push :ignored_line
|
350
347
|
end
|
351
348
|
end
|
@@ -355,8 +352,8 @@ module Rouge
|
|
355
352
|
([ ]+)(!|![\w-]*!) # prefix
|
356
353
|
([ ]+)(!|!?[\w;/?:@&=+$,.!~*'()\[\]%-]+) # tag handle
|
357
354
|
)x do
|
358
|
-
group
|
359
|
-
group
|
355
|
+
group Text; group Keyword::Type
|
356
|
+
group Text; group Keyword::Type
|
360
357
|
pop!; push :ignored_line
|
361
358
|
end
|
362
359
|
end
|
@@ -6,7 +6,7 @@ module Rouge
|
|
6
6
|
module Plugins
|
7
7
|
module Redcarpet
|
8
8
|
def block_code(code, language)
|
9
|
-
lexer = Lexer.find_fancy(language, code) || Lexers::
|
9
|
+
lexer = Lexer.find_fancy(language, code) || Lexers::PlainText
|
10
10
|
|
11
11
|
# XXX HACK: Redcarpet strips hard tabs out of code blocks,
|
12
12
|
# so we assume you're not using leading spaces that aren't tabs,
|
data/lib/rouge/regex_lexer.rb
CHANGED
@@ -165,6 +165,7 @@ module Rouge
|
|
165
165
|
# start_procs.
|
166
166
|
def reset!
|
167
167
|
@stack = nil
|
168
|
+
@current_stream = nil
|
168
169
|
|
169
170
|
self.class.start_procs.each do |pr|
|
170
171
|
instance_eval(&pr)
|
@@ -186,6 +187,8 @@ module Rouge
|
|
186
187
|
def stream_tokens(str, &b)
|
187
188
|
stream = StringScanner.new(str)
|
188
189
|
|
190
|
+
@current_stream = stream
|
191
|
+
|
189
192
|
until stream.eos?
|
190
193
|
debug { "lexer: #{self.class.tag}" }
|
191
194
|
debug { "stack: #{stack.map(&:name).inspect}" }
|
@@ -194,7 +197,7 @@ module Rouge
|
|
194
197
|
|
195
198
|
if !success
|
196
199
|
debug { " no match, yielding Error" }
|
197
|
-
b.call(Token
|
200
|
+
b.call(Token::Tokens::Error, stream.getch)
|
198
201
|
end
|
199
202
|
end
|
200
203
|
end
|
@@ -231,9 +234,7 @@ module Rouge
|
|
231
234
|
def run_callback(stream, callback, &output_stream)
|
232
235
|
with_output_stream(output_stream) do
|
233
236
|
@group_count = 0
|
234
|
-
@last_match = stream
|
235
237
|
instance_exec(stream, &callback)
|
236
|
-
@last_match = nil
|
237
238
|
end
|
238
239
|
end
|
239
240
|
|
@@ -274,18 +275,20 @@ module Rouge
|
|
274
275
|
# (optional) the string value to yield. If absent, this defaults
|
275
276
|
# to the entire last match.
|
276
277
|
def token(tok, val=:__absent__)
|
277
|
-
val = @
|
278
|
-
val
|
279
|
-
|
280
|
-
raise 'no output stream' unless @output_stream
|
281
|
-
|
282
|
-
@output_stream << [Token[tok], val] unless val.empty?
|
278
|
+
val = @current_stream[0] if val == :__absent__
|
279
|
+
yield_token(tok, val)
|
283
280
|
end
|
284
281
|
|
285
282
|
# Yield a token with the next matched group. Subsequent calls
|
286
283
|
# to this method will yield subsequent groups.
|
287
284
|
def group(tok)
|
288
|
-
|
285
|
+
yield_token(tok, @current_stream[@group_count += 1])
|
286
|
+
end
|
287
|
+
|
288
|
+
def groups(*tokens)
|
289
|
+
tokens.each_with_index do |tok, i|
|
290
|
+
yield_token(tok, @current_stream[i+1])
|
291
|
+
end
|
289
292
|
end
|
290
293
|
|
291
294
|
# Delegate the lex to another lexer. The #lex method will be called
|
@@ -299,7 +302,7 @@ module Rouge
|
|
299
302
|
# The text to delegate. This defaults to the last matched string.
|
300
303
|
def delegate(lexer, text=nil)
|
301
304
|
debug { " delegating to #{lexer.inspect}" }
|
302
|
-
text ||= @
|
305
|
+
text ||= @current_stream[0]
|
303
306
|
|
304
307
|
lexer.lex(text, :continue => true) do |tok, val|
|
305
308
|
debug { " delegated token: #{tok.inspect}, #{val.inspect}" }
|
@@ -336,6 +339,12 @@ module Rouge
|
|
336
339
|
nil
|
337
340
|
end
|
338
341
|
|
342
|
+
# replace the head of the stack with the given state
|
343
|
+
def goto(state_name)
|
344
|
+
raise 'empty stack!' if stack.empty?
|
345
|
+
stack[-1] = get_state(state_name)
|
346
|
+
end
|
347
|
+
|
339
348
|
# reset the stack back to `[:root]`.
|
340
349
|
def reset_stack
|
341
350
|
debug { ' resetting stack' }
|
@@ -345,7 +354,10 @@ module Rouge
|
|
345
354
|
|
346
355
|
# Check if `state_name` is in the state stack.
|
347
356
|
def in_state?(state_name)
|
348
|
-
|
357
|
+
state_name = state_name.to_s
|
358
|
+
stack.any? do |state|
|
359
|
+
state.name == state_name.to_s
|
360
|
+
end
|
349
361
|
end
|
350
362
|
|
351
363
|
# Check if `state_name` is the state on top of the state stack.
|
@@ -357,8 +369,8 @@ module Rouge
|
|
357
369
|
def with_output_stream(output_stream, &b)
|
358
370
|
old_output_stream = @output_stream
|
359
371
|
@output_stream = Enumerator::Yielder.new do |tok, val|
|
360
|
-
debug { " yielding #{tok.
|
361
|
-
output_stream.call(
|
372
|
+
debug { " yielding #{tok.qualname}, #{val.inspect}" }
|
373
|
+
output_stream.call(tok, val)
|
362
374
|
end
|
363
375
|
|
364
376
|
yield
|
@@ -366,5 +378,10 @@ module Rouge
|
|
366
378
|
ensure
|
367
379
|
@output_stream = old_output_stream
|
368
380
|
end
|
381
|
+
|
382
|
+
def yield_token(tok, val)
|
383
|
+
return if val.nil? || val.empty?
|
384
|
+
@output_stream.yield(tok, val)
|
385
|
+
end
|
369
386
|
end
|
370
387
|
end
|