parser 0.9.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.autotest +50 -0
- data/.gemtest +0 -0
- data/History.txt +558 -0
- data/Manifest.txt +18 -0
- data/README.txt +87 -0
- data/Rakefile +192 -0
- data/bin/ruby_parse +96 -0
- data/bin/ruby_parse_extract_error +130 -0
- data/lib/gauntlet_rubyparser.rb +117 -0
- data/lib/ruby18_parser.rb +5706 -0
- data/lib/ruby18_parser.y +1846 -0
- data/lib/ruby19_parser.rb +6054 -0
- data/lib/ruby19_parser.y +2035 -0
- data/lib/ruby_lexer.rb +6789 -0
- data/lib/ruby_parser.rb +4 -0
- data/lib/ruby_parser_extras.rb +1148 -0
- data/test/test_ruby_lexer.rb +2028 -0
- data/test/test_ruby_parser.rb +1772 -0
- data/test/test_ruby_parser_extras.rb +228 -0
- metadata +163 -0
@@ -0,0 +1,2028 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# encoding: ascii-8bit
|
3
|
+
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require 'ruby_lexer'
|
6
|
+
require 'ruby_parser_extras'
|
7
|
+
|
8
|
+
class TestRubyLexer < MiniTest::Unit::TestCase
|
9
|
+
alias :deny :refute
|
10
|
+
|
11
|
+
def setup
|
12
|
+
setup_lexer 18
|
13
|
+
end
|
14
|
+
|
15
|
+
def setup_lexer version
|
16
|
+
@lex = RubyLexer.new(version)
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_advance
|
20
|
+
@lex.source = "blah"
|
21
|
+
|
22
|
+
token, = @lex.advance
|
23
|
+
assert token # blah
|
24
|
+
|
25
|
+
token, = @lex.advance
|
26
|
+
deny token # nada
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_read_escape
|
30
|
+
util_escape "\\", "\\"
|
31
|
+
util_escape "\n", "n"
|
32
|
+
util_escape "\t", "t"
|
33
|
+
util_escape "\r", "r"
|
34
|
+
util_escape "\f", "f"
|
35
|
+
util_escape "\13", "v"
|
36
|
+
util_escape "\0", "0"
|
37
|
+
util_escape "\07", "a"
|
38
|
+
util_escape "\007", "a"
|
39
|
+
util_escape "\033", "e"
|
40
|
+
util_escape "\377", "377"
|
41
|
+
util_escape "\377", "xff"
|
42
|
+
util_escape "\010", "b"
|
43
|
+
util_escape " ", "s"
|
44
|
+
util_escape "q", "q" # plain vanilla escape
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_read_escape_c
|
48
|
+
util_escape "\030", "C-x"
|
49
|
+
util_escape "\030", "cx"
|
50
|
+
util_escape "\230", 'C-\M-x'
|
51
|
+
util_escape "\230", 'c\M-x'
|
52
|
+
|
53
|
+
util_escape "\177", "C-?"
|
54
|
+
util_escape "\177", "c?"
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_read_escape_m
|
58
|
+
util_escape "\370", "M-x"
|
59
|
+
util_escape "\230", 'M-\C-x'
|
60
|
+
util_escape "\230", 'M-\cx'
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_read_escape_errors
|
64
|
+
util_escape_bad ""
|
65
|
+
|
66
|
+
util_escape_bad "M"
|
67
|
+
util_escape_bad "M-"
|
68
|
+
util_escape_bad "Mx"
|
69
|
+
|
70
|
+
util_escape_bad "Cx"
|
71
|
+
util_escape_bad "C"
|
72
|
+
util_escape_bad "C-"
|
73
|
+
|
74
|
+
util_escape_bad "c"
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_read_escape_unicode__19
|
78
|
+
util_escape "\xc4\xa3", 'u0123'
|
79
|
+
|
80
|
+
util_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", 'u{123 f0 B123}'
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_read_escape_unicode_bad__19
|
84
|
+
util_escape_bad 'u123'
|
85
|
+
util_escape_bad 'u{}'
|
86
|
+
util_escape_bad 'u{123 f0h}'
|
87
|
+
util_escape_bad 'u{123 f0'
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_yylex_ambiguous_uminus
|
91
|
+
util_lex_token("m -3",
|
92
|
+
:tIDENTIFIER, "m",
|
93
|
+
:tUMINUS_NUM, "-",
|
94
|
+
:tINTEGER, 3)
|
95
|
+
# TODO: verify warning
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_yylex_ambiguous_uplus
|
99
|
+
util_lex_token("m +3",
|
100
|
+
:tIDENTIFIER, "m",
|
101
|
+
:tINTEGER, 3)
|
102
|
+
# TODO: verify warning
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_yylex_and
|
106
|
+
util_lex_token "&", :tAMPER, "&"
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_yylex_and2
|
110
|
+
@lex.state = :expr_end
|
111
|
+
|
112
|
+
util_lex_token "&&", :tANDOP, "&&"
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_yylex_and2_equals
|
116
|
+
@lex.state = :expr_end
|
117
|
+
|
118
|
+
util_lex_token "&&=", :tOP_ASGN, "&&"
|
119
|
+
end
|
120
|
+
|
121
|
+
def test_yylex_and_arg
|
122
|
+
@lex.state = :expr_arg
|
123
|
+
|
124
|
+
util_lex_token(" &y",
|
125
|
+
:tAMPER, "&",
|
126
|
+
:tIDENTIFIER, "y")
|
127
|
+
end
|
128
|
+
|
129
|
+
def test_yylex_and_equals
|
130
|
+
@lex.state = :expr_end
|
131
|
+
|
132
|
+
util_lex_token "&=", :tOP_ASGN, "&"
|
133
|
+
end
|
134
|
+
|
135
|
+
def test_yylex_and_expr
|
136
|
+
@lex.state = :expr_arg
|
137
|
+
|
138
|
+
util_lex_token("x & y",
|
139
|
+
:tIDENTIFIER, "x",
|
140
|
+
:tAMPER2, "&",
|
141
|
+
:tIDENTIFIER, "y")
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_yylex_and_meth
|
145
|
+
util_lex_fname "&", :tAMPER2
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_yylex_assoc
|
149
|
+
util_lex_token "=>", :tASSOC, "=>"
|
150
|
+
end
|
151
|
+
|
152
|
+
def test_yylex_label__18
|
153
|
+
util_lex_token("{a:b",
|
154
|
+
:tLBRACE, "{",
|
155
|
+
:tIDENTIFIER, "a",
|
156
|
+
:tSYMBOL, "b")
|
157
|
+
end
|
158
|
+
|
159
|
+
def test_yylex_label_in_params__18
|
160
|
+
util_lex_token("foo(a:b",
|
161
|
+
:tIDENTIFIER, "foo",
|
162
|
+
:tLPAREN2, "(",
|
163
|
+
:tIDENTIFIER, "a",
|
164
|
+
:tSYMBOL, "b")
|
165
|
+
end
|
166
|
+
|
167
|
+
def test_yylex_label__19
|
168
|
+
setup_lexer 19
|
169
|
+
|
170
|
+
util_lex_token("{a:b",
|
171
|
+
:tLBRACE, "{",
|
172
|
+
:tLABEL, "a",
|
173
|
+
:tIDENTIFIER, "b")
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_yylex_label_in_params__19
|
177
|
+
setup_lexer 19
|
178
|
+
|
179
|
+
util_lex_token("foo(a:b",
|
180
|
+
:tIDENTIFIER, "foo",
|
181
|
+
:tLPAREN2, "(",
|
182
|
+
:tLABEL, "a",
|
183
|
+
:tIDENTIFIER, "b")
|
184
|
+
end
|
185
|
+
|
186
|
+
def test_yylex_command_start__19
|
187
|
+
setup_lexer 19
|
188
|
+
|
189
|
+
%w[case elsif for in until when while
|
190
|
+
if unless and or].each do |keyword|
|
191
|
+
token = "k#{keyword.upcase}".to_sym
|
192
|
+
|
193
|
+
@lex.reset
|
194
|
+
util_lex_token("#{keyword} a:b",
|
195
|
+
token, keyword,
|
196
|
+
:tIDENTIFIER, "a",
|
197
|
+
:tSYMBOL, "b")
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_yylex_mod_not_command_start__19
|
202
|
+
setup_lexer 19
|
203
|
+
|
204
|
+
%w[if unless while until rescue].each do |keyword|
|
205
|
+
token = "k#{keyword.upcase}_MOD".to_sym
|
206
|
+
|
207
|
+
@lex.state = :expr_end
|
208
|
+
util_lex_token("#{keyword} a:b",
|
209
|
+
token, keyword,
|
210
|
+
:tLABEL, "a",
|
211
|
+
:tIDENTIFIER, "b")
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
def test_yylex_back_ref
|
216
|
+
util_lex_token("[$&, $`, $', $+]",
|
217
|
+
:tLBRACK, "[",
|
218
|
+
:tBACK_REF, :"&", :tCOMMA, ",",
|
219
|
+
:tBACK_REF, :"`", :tCOMMA, ",",
|
220
|
+
:tBACK_REF, :"'", :tCOMMA, ",",
|
221
|
+
:tBACK_REF, :"+",
|
222
|
+
:tRBRACK, "]")
|
223
|
+
end
|
224
|
+
|
225
|
+
def test_yylex_backslash
|
226
|
+
util_lex_token("1 \\\n+ 2",
|
227
|
+
:tINTEGER, 1,
|
228
|
+
:tPLUS, "+",
|
229
|
+
:tINTEGER, 2)
|
230
|
+
end
|
231
|
+
|
232
|
+
def test_yylex_backslash_bad
|
233
|
+
util_bad_token("1 \\ + 2",
|
234
|
+
:tINTEGER, 1)
|
235
|
+
end
|
236
|
+
|
237
|
+
def test_yylex_backtick
|
238
|
+
util_lex_token("`ls`",
|
239
|
+
:tXSTRING_BEG, "`",
|
240
|
+
:tSTRING_CONTENT, "ls",
|
241
|
+
:tSTRING_END, "`")
|
242
|
+
end
|
243
|
+
|
244
|
+
def test_yylex_backtick_cmdarg
|
245
|
+
@lex.state = :expr_dot
|
246
|
+
util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
|
247
|
+
|
248
|
+
assert_equal :expr_arg, @lex.state
|
249
|
+
end
|
250
|
+
|
251
|
+
def test_yylex_backtick_dot
|
252
|
+
@lex.state = :expr_dot
|
253
|
+
util_lex_token("a.`(3)",
|
254
|
+
:tIDENTIFIER, "a",
|
255
|
+
:tDOT, ".",
|
256
|
+
:tBACK_REF2, "`",
|
257
|
+
:tLPAREN2, "(",
|
258
|
+
:tINTEGER, 3,
|
259
|
+
:tRPAREN, ")")
|
260
|
+
end
|
261
|
+
|
262
|
+
def test_yylex_backtick_method
|
263
|
+
@lex.state = :expr_fname
|
264
|
+
util_lex_token("`", :tBACK_REF2, "`")
|
265
|
+
assert_equal :expr_end, @lex.state
|
266
|
+
end
|
267
|
+
|
268
|
+
def test_yylex_bad_char
|
269
|
+
util_bad_token(" \010 ")
|
270
|
+
end
|
271
|
+
|
272
|
+
def test_yylex_bang
|
273
|
+
util_lex_token "!", :tBANG, "!"
|
274
|
+
end
|
275
|
+
|
276
|
+
def test_yylex_bang_equals
|
277
|
+
util_lex_token "!=", :tNEQ, "!="
|
278
|
+
end
|
279
|
+
|
280
|
+
def test_yylex_bang_tilde
|
281
|
+
util_lex_token "!~", :tNMATCH, "!~"
|
282
|
+
end
|
283
|
+
|
284
|
+
def test_yylex_carat
|
285
|
+
util_lex_token "^", :tCARET, "^"
|
286
|
+
end
|
287
|
+
|
288
|
+
def test_yylex_carat_equals
|
289
|
+
util_lex_token "^=", :tOP_ASGN, "^"
|
290
|
+
end
|
291
|
+
|
292
|
+
def test_yylex_colon2
|
293
|
+
util_lex_token("A::B",
|
294
|
+
:tCONSTANT, "A",
|
295
|
+
:tCOLON2, "::",
|
296
|
+
:tCONSTANT, "B")
|
297
|
+
end
|
298
|
+
|
299
|
+
def test_yylex_colon3
|
300
|
+
util_lex_token("::Array",
|
301
|
+
:tCOLON3, "::",
|
302
|
+
:tCONSTANT, "Array")
|
303
|
+
end
|
304
|
+
|
305
|
+
def test_yylex_comma
|
306
|
+
util_lex_token ",", :tCOMMA, ","
|
307
|
+
end
|
308
|
+
|
309
|
+
def test_yylex_comment
|
310
|
+
util_lex_token("1 # one\n# two\n2",
|
311
|
+
:tINTEGER, 1,
|
312
|
+
:tNL, nil,
|
313
|
+
:tINTEGER, 2)
|
314
|
+
assert_equal "# one\n# two\n", @lex.comments
|
315
|
+
end
|
316
|
+
|
317
|
+
def test_yylex_comment_begin
|
318
|
+
util_lex_token("=begin\nblah\nblah\n=end\n42",
|
319
|
+
:tINTEGER, 42)
|
320
|
+
assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
|
321
|
+
end
|
322
|
+
|
323
|
+
def test_yylex_comment_begin_bad
|
324
|
+
util_bad_token("=begin\nblah\nblah\n")
|
325
|
+
assert_equal "", @lex.comments
|
326
|
+
end
|
327
|
+
|
328
|
+
def test_yylex_comment_begin_not_comment
|
329
|
+
util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
|
330
|
+
:tIDENTIFIER, "beginfoo",
|
331
|
+
:tEQL, "=",
|
332
|
+
:tINTEGER, 5,
|
333
|
+
:tNL, nil,
|
334
|
+
:tIDENTIFIER, "p",
|
335
|
+
:tIDENTIFIER, "x",
|
336
|
+
:tEQL, "=",
|
337
|
+
:tIDENTIFIER, "beginfoo")
|
338
|
+
end
|
339
|
+
|
340
|
+
def test_yylex_comment_begin_space
|
341
|
+
util_lex_token("=begin blah\nblah\n=end\n")
|
342
|
+
assert_equal "=begin blah\nblah\n=end\n", @lex.comments
|
343
|
+
end
|
344
|
+
|
345
|
+
def test_yylex_comment_end_space_and_text
|
346
|
+
util_lex_token("=begin blah\nblah\n=end blab\n")
|
347
|
+
assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
|
348
|
+
end
|
349
|
+
|
350
|
+
def test_yylex_comment_eos
|
351
|
+
util_lex_token("# comment")
|
352
|
+
end
|
353
|
+
|
354
|
+
def test_yylex_constant
|
355
|
+
util_lex_token("ArgumentError",
|
356
|
+
:tCONSTANT, "ArgumentError")
|
357
|
+
end
|
358
|
+
|
359
|
+
def test_yylex_constant_semi
|
360
|
+
util_lex_token("ArgumentError;",
|
361
|
+
:tCONSTANT, "ArgumentError",
|
362
|
+
:tSEMI, ";")
|
363
|
+
end
|
364
|
+
|
365
|
+
def test_yylex_cvar
|
366
|
+
util_lex_token "@@blah", :tCVAR, "@@blah"
|
367
|
+
end
|
368
|
+
|
369
|
+
def test_yylex_cvar_bad
|
370
|
+
assert_raises RubyParser::SyntaxError do
|
371
|
+
util_lex_token "@@1"
|
372
|
+
end
|
373
|
+
end
|
374
|
+
|
375
|
+
def test_yylex_div
|
376
|
+
util_lex_token("a / 2",
|
377
|
+
:tIDENTIFIER, "a",
|
378
|
+
:tDIVIDE, "/",
|
379
|
+
:tINTEGER, 2)
|
380
|
+
end
|
381
|
+
|
382
|
+
def test_yylex_div_equals
|
383
|
+
util_lex_token("a /= 2",
|
384
|
+
:tIDENTIFIER, "a",
|
385
|
+
:tOP_ASGN, "/",
|
386
|
+
:tINTEGER, 2)
|
387
|
+
end
|
388
|
+
|
389
|
+
def test_yylex_do
|
390
|
+
util_lex_token("x do 42 end",
|
391
|
+
:tIDENTIFIER, "x",
|
392
|
+
:kDO, "do",
|
393
|
+
:tINTEGER, 42,
|
394
|
+
:kEND, "end")
|
395
|
+
end
|
396
|
+
|
397
|
+
def test_yylex_do_block
|
398
|
+
@lex.state = :expr_endarg
|
399
|
+
|
400
|
+
util_lex_token("do 42 end",
|
401
|
+
:kDO_BLOCK, "do",
|
402
|
+
:tINTEGER, 42,
|
403
|
+
:kEND, "end")
|
404
|
+
end
|
405
|
+
|
406
|
+
def test_yylex_do_cond
|
407
|
+
@lex.cond.push true
|
408
|
+
|
409
|
+
util_lex_token("x do 42 end",
|
410
|
+
:tIDENTIFIER, "x",
|
411
|
+
:kDO_COND, "do",
|
412
|
+
:tINTEGER, 42,
|
413
|
+
:kEND, "end")
|
414
|
+
end
|
415
|
+
|
416
|
+
def test_yylex_dot
|
417
|
+
util_lex_token ".", :tDOT, "."
|
418
|
+
end
|
419
|
+
|
420
|
+
def test_yylex_dot2
|
421
|
+
util_lex_token "..", :tDOT2, ".."
|
422
|
+
end
|
423
|
+
|
424
|
+
def test_yylex_dot3
|
425
|
+
util_lex_token "...", :tDOT3, "..."
|
426
|
+
end
|
427
|
+
|
428
|
+
def test_yylex_equals
|
429
|
+
util_lex_token "=", :tEQL, "="
|
430
|
+
end
|
431
|
+
|
432
|
+
def test_yylex_equals2
|
433
|
+
util_lex_token "==", :tEQ, "=="
|
434
|
+
end
|
435
|
+
|
436
|
+
def test_yylex_equals3
|
437
|
+
util_lex_token "===", :tEQQ, "==="
|
438
|
+
end
|
439
|
+
|
440
|
+
def test_yylex_equals_tilde
|
441
|
+
util_lex_token "=~", :tMATCH, "=~"
|
442
|
+
end
|
443
|
+
|
444
|
+
def test_yylex_float
|
445
|
+
util_lex_token "1.0", :tFLOAT, 1.0
|
446
|
+
end
|
447
|
+
|
448
|
+
def test_yylex_float_bad_no_underscores
|
449
|
+
util_bad_token "1__0.0"
|
450
|
+
end
|
451
|
+
|
452
|
+
def test_yylex_float_bad_no_zero_leading
|
453
|
+
util_bad_token ".0"
|
454
|
+
end
|
455
|
+
|
456
|
+
def test_yylex_float_bad_trailing_underscore
|
457
|
+
util_bad_token "123_.0"
|
458
|
+
end
|
459
|
+
|
460
|
+
def test_yylex_float_call
|
461
|
+
util_lex_token("1.0.to_s",
|
462
|
+
:tFLOAT, 1.0,
|
463
|
+
:tDOT, ".",
|
464
|
+
:tIDENTIFIER, "to_s")
|
465
|
+
end
|
466
|
+
|
467
|
+
def test_yylex_float_dot_E
|
468
|
+
util_lex_token "1.0E10", :tFLOAT, 1.0e10
|
469
|
+
end
|
470
|
+
|
471
|
+
def test_yylex_float_dot_E_neg
|
472
|
+
util_lex_token("-1.0E10",
|
473
|
+
:tUMINUS_NUM, "-",
|
474
|
+
:tFLOAT, 1.0e10)
|
475
|
+
end
|
476
|
+
|
477
|
+
def test_yylex_float_dot_e
|
478
|
+
util_lex_token "1.0e10", :tFLOAT, 1.0e10
|
479
|
+
end
|
480
|
+
|
481
|
+
def test_yylex_float_dot_e_neg
|
482
|
+
util_lex_token("-1.0e10",
|
483
|
+
:tUMINUS_NUM, "-",
|
484
|
+
:tFLOAT, 1.0e10)
|
485
|
+
end
|
486
|
+
|
487
|
+
def test_yylex_float_e
|
488
|
+
util_lex_token "1e10", :tFLOAT, 1e10
|
489
|
+
end
|
490
|
+
|
491
|
+
def test_yylex_float_e_bad_trailing_underscore
|
492
|
+
util_bad_token "123_e10"
|
493
|
+
end
|
494
|
+
|
495
|
+
def test_yylex_float_e_minus
|
496
|
+
util_lex_token "1e-10", :tFLOAT, 1e-10
|
497
|
+
end
|
498
|
+
|
499
|
+
def test_yylex_float_e_neg
|
500
|
+
util_lex_token("-1e10",
|
501
|
+
:tUMINUS_NUM, "-",
|
502
|
+
:tFLOAT, 1e10)
|
503
|
+
end
|
504
|
+
|
505
|
+
def test_yylex_float_e_neg_minus
|
506
|
+
util_lex_token("-1e-10",
|
507
|
+
:tUMINUS_NUM, "-",
|
508
|
+
:tFLOAT, 1e-10)
|
509
|
+
end
|
510
|
+
|
511
|
+
def test_yylex_float_e_neg_plus
|
512
|
+
util_lex_token("-1e+10",
|
513
|
+
:tUMINUS_NUM, "-",
|
514
|
+
:tFLOAT, 1e10)
|
515
|
+
end
|
516
|
+
|
517
|
+
def test_yylex_float_e_plus
|
518
|
+
util_lex_token "1e+10", :tFLOAT, 1e10
|
519
|
+
end
|
520
|
+
|
521
|
+
def test_yylex_float_e_zero
|
522
|
+
util_lex_token "0e0", :tFLOAT, 0e0
|
523
|
+
end
|
524
|
+
|
525
|
+
def test_yylex_float_neg
|
526
|
+
util_lex_token("-1.0",
|
527
|
+
:tUMINUS_NUM, "-",
|
528
|
+
:tFLOAT, 1.0)
|
529
|
+
end
|
530
|
+
|
531
|
+
def test_yylex_ge
|
532
|
+
util_lex_token("a >= 2",
|
533
|
+
:tIDENTIFIER, "a",
|
534
|
+
:tGEQ, ">=",
|
535
|
+
:tINTEGER, 2)
|
536
|
+
end
|
537
|
+
|
538
|
+
def test_yylex_global
|
539
|
+
util_lex_token("$blah", :tGVAR, "$blah")
|
540
|
+
end
|
541
|
+
|
542
|
+
def test_yylex_global_backref
|
543
|
+
@lex.state = :expr_fname
|
544
|
+
util_lex_token("$`", :tGVAR, "$`")
|
545
|
+
end
|
546
|
+
|
547
|
+
def test_yylex_global_dash_nothing
|
548
|
+
util_lex_token("$- ", :tGVAR, "$-")
|
549
|
+
end
|
550
|
+
|
551
|
+
def test_yylex_global_dash_something
|
552
|
+
util_lex_token("$-x", :tGVAR, "$-x")
|
553
|
+
end
|
554
|
+
|
555
|
+
def test_yylex_global_number
|
556
|
+
@lex.state = :expr_fname
|
557
|
+
util_lex_token("$1", :tGVAR, "$1")
|
558
|
+
end
|
559
|
+
|
560
|
+
def test_yylex_global_number_big
|
561
|
+
@lex.state = :expr_fname
|
562
|
+
util_lex_token("$1234", :tGVAR, "$1234")
|
563
|
+
end
|
564
|
+
|
565
|
+
def test_yylex_global_other
|
566
|
+
util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
|
567
|
+
:tLBRACK, "[",
|
568
|
+
:tGVAR, "$~", :tCOMMA, ",",
|
569
|
+
:tGVAR, "$*", :tCOMMA, ",",
|
570
|
+
:tGVAR, "$$", :tCOMMA, ",",
|
571
|
+
:tGVAR, "$\?", :tCOMMA, ",",
|
572
|
+
:tGVAR, "$!", :tCOMMA, ",",
|
573
|
+
:tGVAR, "$@", :tCOMMA, ",",
|
574
|
+
:tGVAR, "$/", :tCOMMA, ",",
|
575
|
+
:tGVAR, "$\\", :tCOMMA, ",",
|
576
|
+
:tGVAR, "$;", :tCOMMA, ",",
|
577
|
+
:tGVAR, "$,", :tCOMMA, ",",
|
578
|
+
:tGVAR, "$.", :tCOMMA, ",",
|
579
|
+
:tGVAR, "$=", :tCOMMA, ",",
|
580
|
+
:tGVAR, "$:", :tCOMMA, ",",
|
581
|
+
:tGVAR, "$<", :tCOMMA, ",",
|
582
|
+
:tGVAR, "$>", :tCOMMA, ",",
|
583
|
+
:tGVAR, "$\"",
|
584
|
+
:tRBRACK, "]")
|
585
|
+
end
|
586
|
+
|
587
|
+
def test_yylex_global_underscore
|
588
|
+
util_lex_token("$_",
|
589
|
+
:tGVAR, "$_")
|
590
|
+
end
|
591
|
+
|
592
|
+
def test_yylex_global_wierd
|
593
|
+
util_lex_token("$__blah",
|
594
|
+
:tGVAR, "$__blah")
|
595
|
+
end
|
596
|
+
|
597
|
+
def test_yylex_global_zero
|
598
|
+
util_lex_token("$0", :tGVAR, "$0")
|
599
|
+
end
|
600
|
+
|
601
|
+
def test_yylex_gt
|
602
|
+
util_lex_token("a > 2",
|
603
|
+
:tIDENTIFIER, "a",
|
604
|
+
:tGT, ">",
|
605
|
+
:tINTEGER, 2)
|
606
|
+
end
|
607
|
+
|
608
|
+
def test_yylex_heredoc_backtick
|
609
|
+
util_lex_token("a = <<`EOF`\n blah blah\nEOF\n",
|
610
|
+
:tIDENTIFIER, "a",
|
611
|
+
:tEQL, "=",
|
612
|
+
:tXSTRING_BEG, "`",
|
613
|
+
:tSTRING_CONTENT, " blah blah\n",
|
614
|
+
:tSTRING_END, "EOF",
|
615
|
+
:tNL, nil)
|
616
|
+
end
|
617
|
+
|
618
|
+
def test_yylex_heredoc_double
|
619
|
+
util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n",
|
620
|
+
:tIDENTIFIER, "a",
|
621
|
+
:tEQL, "=",
|
622
|
+
:tSTRING_BEG, "\"",
|
623
|
+
:tSTRING_CONTENT, " blah blah\n",
|
624
|
+
:tSTRING_END, "EOF",
|
625
|
+
:tNL, nil)
|
626
|
+
end
|
627
|
+
|
628
|
+
def test_yylex_heredoc_double_dash
|
629
|
+
util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n",
|
630
|
+
:tIDENTIFIER, "a",
|
631
|
+
:tEQL, "=",
|
632
|
+
:tSTRING_BEG, "\"",
|
633
|
+
:tSTRING_CONTENT, " blah blah\n",
|
634
|
+
:tSTRING_END, "EOF",
|
635
|
+
:tNL, nil)
|
636
|
+
end
|
637
|
+
|
638
|
+
def test_yylex_heredoc_double_eos
|
639
|
+
util_bad_token("a = <<\"EOF\"\nblah",
|
640
|
+
:tIDENTIFIER, "a",
|
641
|
+
:tEQL, "=",
|
642
|
+
:tSTRING_BEG, "\"")
|
643
|
+
end
|
644
|
+
|
645
|
+
def test_yylex_heredoc_double_eos_nl
|
646
|
+
util_bad_token("a = <<\"EOF\"\nblah\n",
|
647
|
+
:tIDENTIFIER, "a",
|
648
|
+
:tEQL, "=",
|
649
|
+
:tSTRING_BEG, "\"")
|
650
|
+
end
|
651
|
+
|
652
|
+
def test_yylex_heredoc_double_interp
|
653
|
+
util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
|
654
|
+
:tIDENTIFIER, "a",
|
655
|
+
:tEQL, "=",
|
656
|
+
:tSTRING_BEG, "\"",
|
657
|
+
:tSTRING_CONTENT, "#x a ",
|
658
|
+
:tSTRING_DVAR, nil,
|
659
|
+
:tIVAR, "@a",
|
660
|
+
:tSTRING_CONTENT, " b ",
|
661
|
+
:tSTRING_DVAR, nil,
|
662
|
+
:tGVAR, "$b",
|
663
|
+
:tSTRING_CONTENT, " c ",
|
664
|
+
:tSTRING_DBEG, '#{',
|
665
|
+
:tINTEGER, 3,
|
666
|
+
:tRCURLY, "}",
|
667
|
+
:tSTRING_CONTENT, " \n",
|
668
|
+
:tSTRING_END, "EOF",
|
669
|
+
:tNL, nil)
|
670
|
+
end
|
671
|
+
|
672
|
+
def test_yylex_heredoc_empty
|
673
|
+
util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
|
674
|
+
:tSTRING_BEG, "\"",
|
675
|
+
:tSTRING_DBEG, "\#{",
|
676
|
+
:tIDENTIFIER, "x",
|
677
|
+
:tRCURLY, "}",
|
678
|
+
:tSTRING_CONTENT, "\n",
|
679
|
+
:tSTRING_CONTENT, "blah2\n",
|
680
|
+
:tSTRING_END, "",
|
681
|
+
:tNL, nil)
|
682
|
+
end
|
683
|
+
|
684
|
+
def test_yylex_heredoc_none
|
685
|
+
util_lex_token("a = <<EOF\nblah\nblah\nEOF",
|
686
|
+
:tIDENTIFIER, "a",
|
687
|
+
:tEQL, "=",
|
688
|
+
:tSTRING_BEG, "\"",
|
689
|
+
:tSTRING_CONTENT, "blah\n",
|
690
|
+
:tSTRING_CONTENT, "blah\n",
|
691
|
+
:tSTRING_END, "EOF",
|
692
|
+
:tNL, nil)
|
693
|
+
end
|
694
|
+
|
695
|
+
def test_yylex_heredoc_none_dash
|
696
|
+
util_lex_token("a = <<-EOF\nblah\nblah\n EOF",
|
697
|
+
:tIDENTIFIER, "a",
|
698
|
+
:tEQL, "=",
|
699
|
+
:tSTRING_BEG, "\"",
|
700
|
+
:tSTRING_CONTENT, "blah\n",
|
701
|
+
:tSTRING_CONTENT, "blah\n",
|
702
|
+
:tSTRING_END, "EOF",
|
703
|
+
:tNL, nil)
|
704
|
+
end
|
705
|
+
|
706
|
+
def test_yylex_heredoc_single
|
707
|
+
util_lex_token("a = <<'EOF'\n blah blah\nEOF\n",
|
708
|
+
:tIDENTIFIER, "a",
|
709
|
+
:tEQL, "=",
|
710
|
+
:tSTRING_BEG, "'",
|
711
|
+
:tSTRING_CONTENT, " blah blah\n",
|
712
|
+
:tSTRING_END, "EOF",
|
713
|
+
:tNL, nil)
|
714
|
+
end
|
715
|
+
|
716
|
+
def test_yylex_heredoc_single_bad_eos_body
|
717
|
+
util_bad_token("a = <<'EOF'\nblah",
|
718
|
+
:tIDENTIFIER, "a",
|
719
|
+
:tEQL, "=",
|
720
|
+
:tSTRING_BEG, "'")
|
721
|
+
end
|
722
|
+
|
723
|
+
def test_yylex_heredoc_single_dash
|
724
|
+
util_lex_token("a = <<-'EOF'\n blah blah\n EOF\n",
|
725
|
+
:tIDENTIFIER, "a",
|
726
|
+
:tEQL, "=",
|
727
|
+
:tSTRING_BEG, "'",
|
728
|
+
:tSTRING_CONTENT, " blah blah\n",
|
729
|
+
:tSTRING_END, "EOF",
|
730
|
+
:tNL, nil)
|
731
|
+
end
|
732
|
+
|
733
|
+
def test_yylex_identifier
|
734
|
+
util_lex_token("identifier", :tIDENTIFIER, "identifier")
|
735
|
+
end
|
736
|
+
|
737
|
+
def test_yylex_identifier_bang
|
738
|
+
util_lex_token("identifier!", :tFID, "identifier!")
|
739
|
+
end
|
740
|
+
|
741
|
+
def test_yylex_identifier_cmp
|
742
|
+
util_lex_fname "<=>", :tCMP
|
743
|
+
end
|
744
|
+
|
745
|
+
def test_yylex_identifier_def
|
746
|
+
util_lex_fname "identifier", :tIDENTIFIER, :expr_end
|
747
|
+
end
|
748
|
+
|
749
|
+
def test_yylex_identifier_eh
|
750
|
+
util_lex_token("identifier?", :tFID, "identifier?")
|
751
|
+
end
|
752
|
+
|
753
|
+
def test_yylex_identifier_equals_arrow
|
754
|
+
util_lex_token(":blah==>",
|
755
|
+
:tSYMBOL, "blah=",
|
756
|
+
:tASSOC, "=>")
|
757
|
+
end
|
758
|
+
|
759
|
+
def test_yylex_identifier_equals3
|
760
|
+
util_lex_token(":a===b",
|
761
|
+
:tSYMBOL, "a",
|
762
|
+
:tEQQ, "===",
|
763
|
+
:tIDENTIFIER, "b")
|
764
|
+
end
|
765
|
+
|
766
|
+
def test_yylex_identifier_equals_equals_arrow
|
767
|
+
util_lex_token(":a==>b",
|
768
|
+
:tSYMBOL, "a=",
|
769
|
+
:tASSOC, "=>",
|
770
|
+
:tIDENTIFIER, "b")
|
771
|
+
end
|
772
|
+
|
773
|
+
def test_yylex_identifier_equals_caret
|
774
|
+
util_lex_fname "^", :tCARET
|
775
|
+
end
|
776
|
+
|
777
|
+
def test_yylex_identifier_equals_def
|
778
|
+
util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
|
779
|
+
end
|
780
|
+
|
781
|
+
def test_yylex_identifier_equals_def2
|
782
|
+
util_lex_fname "==", :tEQ
|
783
|
+
end
|
784
|
+
|
785
|
+
def test_yylex_identifier_equals_expr
|
786
|
+
@lex.state = :expr_dot
|
787
|
+
util_lex_token("y = arg",
|
788
|
+
:tIDENTIFIER, "y",
|
789
|
+
:tEQL, "=",
|
790
|
+
:tIDENTIFIER, "arg")
|
791
|
+
|
792
|
+
assert_equal :expr_arg, @lex.state
|
793
|
+
end
|
794
|
+
|
795
|
+
def test_yylex_identifier_equals_or
|
796
|
+
util_lex_fname "|", :tPIPE
|
797
|
+
end
|
798
|
+
|
799
|
+
def test_yylex_identifier_equals_slash
|
800
|
+
util_lex_fname "/", :tDIVIDE
|
801
|
+
end
|
802
|
+
|
803
|
+
def test_yylex_identifier_equals_tilde
|
804
|
+
@lex.state = :expr_fname # can only set via parser's defs
|
805
|
+
util_lex_token("identifier=~",
|
806
|
+
:tIDENTIFIER, "identifier",
|
807
|
+
:tMATCH, "=~")
|
808
|
+
end
|
809
|
+
|
810
|
+
def test_yylex_identifier_gt
|
811
|
+
util_lex_fname ">", :tGT
|
812
|
+
end
|
813
|
+
|
814
|
+
def test_yylex_identifier_le
|
815
|
+
util_lex_fname "<=", :tLEQ
|
816
|
+
end
|
817
|
+
|
818
|
+
def test_yylex_identifier_lt
|
819
|
+
util_lex_fname "<", :tLT
|
820
|
+
end
|
821
|
+
|
822
|
+
def test_yylex_identifier_tilde
|
823
|
+
util_lex_fname "~", :tTILDE
|
824
|
+
end
|
825
|
+
|
826
|
+
def test_yylex_index
|
827
|
+
util_lex_fname "[]", :tAREF
|
828
|
+
end
|
829
|
+
|
830
|
+
def test_yylex_index_equals
|
831
|
+
util_lex_fname "[]=", :tASET
|
832
|
+
end
|
833
|
+
|
834
|
+
def test_yylex_integer
|
835
|
+
util_lex_token "42", :tINTEGER, 42
|
836
|
+
end
|
837
|
+
|
838
|
+
def test_yylex_integer_bin
|
839
|
+
util_lex_token "0b101010", :tINTEGER, 42
|
840
|
+
end
|
841
|
+
|
842
|
+
def test_yylex_integer_bin_bad_none
|
843
|
+
util_bad_token "0b "
|
844
|
+
end
|
845
|
+
|
846
|
+
def test_yylex_integer_bin_bad_underscores
|
847
|
+
util_bad_token "0b10__01"
|
848
|
+
end
|
849
|
+
|
850
|
+
def test_yylex_integer_dec
|
851
|
+
util_lex_token "42", :tINTEGER, 42
|
852
|
+
end
|
853
|
+
|
854
|
+
def test_yylex_integer_dec_bad_underscores
|
855
|
+
util_bad_token "42__24"
|
856
|
+
end
|
857
|
+
|
858
|
+
def test_yylex_integer_dec_d
|
859
|
+
util_lex_token "0d42", :tINTEGER, 42
|
860
|
+
end
|
861
|
+
|
862
|
+
def test_yylex_integer_dec_d_bad_none
|
863
|
+
util_bad_token "0d"
|
864
|
+
end
|
865
|
+
|
866
|
+
def test_yylex_integer_dec_d_bad_underscores
|
867
|
+
util_bad_token "0d42__24"
|
868
|
+
end
|
869
|
+
|
870
|
+
def test_yylex_question_eh_a__18
|
871
|
+
@lex = RubyLexer.new 18
|
872
|
+
|
873
|
+
util_lex_token "?a", :tINTEGER, 97
|
874
|
+
end
|
875
|
+
|
876
|
+
def test_yylex_question_eh_a__19
|
877
|
+
@lex = RubyLexer.new 19
|
878
|
+
|
879
|
+
util_lex_token '?a', :tSTRING, "a"
|
880
|
+
end
|
881
|
+
|
882
|
+
def test_yylex_question_eh_escape_M_escape_C__18
|
883
|
+
@lex = RubyLexer.new 18
|
884
|
+
|
885
|
+
util_lex_token '?\M-\C-a', :tINTEGER, 129
|
886
|
+
end
|
887
|
+
|
888
|
+
def test_yylex_question_eh_escape_M_escape_C__19
|
889
|
+
@lex = RubyLexer.new 19
|
890
|
+
|
891
|
+
util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
|
892
|
+
end
|
893
|
+
|
894
|
+
def test_yylex_integer_hex
|
895
|
+
util_lex_token "0x2a", :tINTEGER, 42
|
896
|
+
end
|
897
|
+
|
898
|
+
def test_yylex_integer_hex_bad_none
|
899
|
+
util_bad_token "0x "
|
900
|
+
end
|
901
|
+
|
902
|
+
def test_yylex_integer_hex_bad_underscores
|
903
|
+
util_bad_token "0xab__cd"
|
904
|
+
end
|
905
|
+
|
906
|
+
def test_yylex_integer_oct
|
907
|
+
util_lex_token "052", :tINTEGER, 42
|
908
|
+
end
|
909
|
+
|
910
|
+
def test_yylex_integer_oct_bad_range
|
911
|
+
util_bad_token "08"
|
912
|
+
end
|
913
|
+
|
914
|
+
def test_yylex_integer_oct_bad_underscores
|
915
|
+
util_bad_token "01__23"
|
916
|
+
end
|
917
|
+
|
918
|
+
def test_yylex_integer_oct_O
|
919
|
+
util_lex_token "0O52", :tINTEGER, 42
|
920
|
+
end
|
921
|
+
|
922
|
+
def test_yylex_integer_oct_O_bad_range
|
923
|
+
util_bad_token "0O8"
|
924
|
+
end
|
925
|
+
|
926
|
+
def test_yylex_integer_oct_O_bad_underscores
|
927
|
+
util_bad_token "0O1__23"
|
928
|
+
end
|
929
|
+
|
930
|
+
def test_yylex_integer_oct_O_not_bad_none
|
931
|
+
util_lex_token "0O ", :tINTEGER, 0
|
932
|
+
end
|
933
|
+
|
934
|
+
def test_yylex_integer_oct_o
|
935
|
+
util_lex_token "0o52", :tINTEGER, 42
|
936
|
+
end
|
937
|
+
|
938
|
+
def test_yylex_integer_oct_o_bad_range
|
939
|
+
util_bad_token "0o8"
|
940
|
+
end
|
941
|
+
|
942
|
+
def test_yylex_integer_oct_o_bad_underscores
|
943
|
+
util_bad_token "0o1__23"
|
944
|
+
end
|
945
|
+
|
946
|
+
def test_yylex_integer_oct_o_not_bad_none
|
947
|
+
util_lex_token "0o ", :tINTEGER, 0
|
948
|
+
end
|
949
|
+
|
950
|
+
def test_yylex_integer_trailing
|
951
|
+
util_lex_token("1.to_s",
|
952
|
+
:tINTEGER, 1,
|
953
|
+
:tDOT, '.',
|
954
|
+
:tIDENTIFIER, 'to_s')
|
955
|
+
end
|
956
|
+
|
957
|
+
def test_yylex_integer_underscore
|
958
|
+
util_lex_token "4_2", :tINTEGER, 42
|
959
|
+
end
|
960
|
+
|
961
|
+
def test_yylex_integer_underscore_bad
|
962
|
+
util_bad_token "4__2"
|
963
|
+
end
|
964
|
+
|
965
|
+
def test_yylex_integer_zero
|
966
|
+
util_lex_token "0", :tINTEGER, 0
|
967
|
+
end
|
968
|
+
|
969
|
+
def test_yylex_ivar
|
970
|
+
util_lex_token "@blah", :tIVAR, "@blah"
|
971
|
+
end
|
972
|
+
|
973
|
+
def test_yylex_ivar_bad
|
974
|
+
util_bad_token "@1"
|
975
|
+
end
|
976
|
+
|
977
|
+
def test_yylex_ivar_bad_0_length
|
978
|
+
util_bad_token "1+@\n", :tINTEGER, 1, :tPLUS, "+"
|
979
|
+
end
|
980
|
+
|
981
|
+
def test_yylex_keyword_expr
|
982
|
+
@lex.state = :expr_endarg
|
983
|
+
|
984
|
+
util_lex_token("if", :kIF_MOD, "if")
|
985
|
+
|
986
|
+
assert_equal :expr_beg, @lex.state
|
987
|
+
end
|
988
|
+
|
989
|
+
def test_yylex_lt
|
990
|
+
util_lex_token "<", :tLT, "<"
|
991
|
+
end
|
992
|
+
|
993
|
+
def test_yylex_lt2
|
994
|
+
util_lex_token("a <\< b",
|
995
|
+
:tIDENTIFIER, "a",
|
996
|
+
:tLSHFT, "<\<",
|
997
|
+
:tIDENTIFIER, "b")
|
998
|
+
|
999
|
+
end
|
1000
|
+
|
1001
|
+
def test_yylex_lt2_equals
|
1002
|
+
util_lex_token("a <\<= b",
|
1003
|
+
:tIDENTIFIER, "a",
|
1004
|
+
:tOP_ASGN, "<\<",
|
1005
|
+
:tIDENTIFIER, "b")
|
1006
|
+
end
|
1007
|
+
|
1008
|
+
def test_yylex_lt_equals
|
1009
|
+
util_lex_token "<=", :tLEQ, "<="
|
1010
|
+
end
|
1011
|
+
|
1012
|
+
def test_yylex_minus
|
1013
|
+
util_lex_token("1 - 2",
|
1014
|
+
:tINTEGER, 1,
|
1015
|
+
:tMINUS, "-",
|
1016
|
+
:tINTEGER, 2)
|
1017
|
+
end
|
1018
|
+
|
1019
|
+
def test_yylex_minus_equals
|
1020
|
+
@lex.state = :expr_end
|
1021
|
+
|
1022
|
+
util_lex_token "-=", :tOP_ASGN, "-"
|
1023
|
+
end
|
1024
|
+
|
1025
|
+
def test_yylex_minus_method
|
1026
|
+
@lex.state = :expr_fname
|
1027
|
+
util_lex_token "-", :tMINUS, "-"
|
1028
|
+
end
|
1029
|
+
|
1030
|
+
def test_yylex_minus_unary_method
|
1031
|
+
@lex.state = :expr_fname
|
1032
|
+
util_lex_token "-@", :tUMINUS, "-@"
|
1033
|
+
end
|
1034
|
+
|
1035
|
+
def test_yylex_minus_unary_number
|
1036
|
+
util_lex_token("-42",
|
1037
|
+
:tUMINUS_NUM, "-",
|
1038
|
+
:tINTEGER, 42)
|
1039
|
+
end
|
1040
|
+
|
1041
|
+
def test_yylex_nth_ref
|
1042
|
+
util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
|
1043
|
+
:tLBRACK, "[",
|
1044
|
+
:tNTH_REF, 1, :tCOMMA, ",",
|
1045
|
+
:tNTH_REF, 2, :tCOMMA, ",",
|
1046
|
+
:tNTH_REF, 3, :tCOMMA, ",",
|
1047
|
+
:tNTH_REF, 4, :tCOMMA, ",",
|
1048
|
+
:tNTH_REF, 5, :tCOMMA, ",",
|
1049
|
+
:tNTH_REF, 6, :tCOMMA, ",",
|
1050
|
+
:tNTH_REF, 7, :tCOMMA, ",",
|
1051
|
+
:tNTH_REF, 8, :tCOMMA, ",",
|
1052
|
+
:tNTH_REF, 9,
|
1053
|
+
:tRBRACK, "]")
|
1054
|
+
end
|
1055
|
+
|
1056
|
+
def test_yylex_open_bracket
|
1057
|
+
util_lex_token("(", :tLPAREN, "(")
|
1058
|
+
end
|
1059
|
+
|
1060
|
+
def test_yylex_open_bracket_cmdarg
|
1061
|
+
util_lex_token("m (", :tIDENTIFIER, "m",
|
1062
|
+
:tLPAREN_ARG, "(")
|
1063
|
+
end
|
1064
|
+
|
1065
|
+
def test_yylex_open_bracket_exprarg
|
1066
|
+
util_lex_token("m(", :tIDENTIFIER, "m",
|
1067
|
+
:tLPAREN2, "(")
|
1068
|
+
end
|
1069
|
+
|
1070
|
+
def test_yylex_open_curly_bracket
|
1071
|
+
util_lex_token("{",
|
1072
|
+
:tLBRACE, "{")
|
1073
|
+
end
|
1074
|
+
|
1075
|
+
def test_yylex_open_curly_bracket_arg
|
1076
|
+
util_lex_token("m { 3 }",
|
1077
|
+
:tIDENTIFIER, "m",
|
1078
|
+
:tLCURLY, "{",
|
1079
|
+
:tINTEGER, 3,
|
1080
|
+
:tRCURLY, "}")
|
1081
|
+
end
|
1082
|
+
|
1083
|
+
def test_yylex_open_curly_bracket_block
|
1084
|
+
@lex.state = :expr_endarg # seen m(3)
|
1085
|
+
|
1086
|
+
util_lex_token("{ 4 }",
|
1087
|
+
:tLBRACE_ARG, "{",
|
1088
|
+
:tINTEGER, 4,
|
1089
|
+
:tRCURLY, "}")
|
1090
|
+
end
|
1091
|
+
|
1092
|
+
def test_yylex_open_square_bracket_arg
|
1093
|
+
util_lex_token("m [ 3 ]",
|
1094
|
+
:tIDENTIFIER, "m",
|
1095
|
+
:tLBRACK, "[",
|
1096
|
+
:tINTEGER, 3,
|
1097
|
+
:tRBRACK, "]")
|
1098
|
+
end
|
1099
|
+
|
1100
|
+
def test_yylex_open_square_bracket_ary
|
1101
|
+
util_lex_token("[1, 2, 3]",
|
1102
|
+
:tLBRACK, "[",
|
1103
|
+
:tINTEGER, 1,
|
1104
|
+
:tCOMMA, ",",
|
1105
|
+
:tINTEGER, 2,
|
1106
|
+
:tCOMMA, ",",
|
1107
|
+
:tINTEGER, 3,
|
1108
|
+
:tRBRACK, "]")
|
1109
|
+
end
|
1110
|
+
|
1111
|
+
def test_yylex_open_square_bracket_meth
|
1112
|
+
util_lex_token("m[3]",
|
1113
|
+
:tIDENTIFIER, "m",
|
1114
|
+
:tLBRACK2, "[",
|
1115
|
+
:tINTEGER, 3,
|
1116
|
+
:tRBRACK, "]")
|
1117
|
+
end
|
1118
|
+
|
1119
|
+
def test_yylex_or
|
1120
|
+
util_lex_token "|", :tPIPE, "|"
|
1121
|
+
end
|
1122
|
+
|
1123
|
+
def test_yylex_or2
|
1124
|
+
util_lex_token "||", :tOROP, "||"
|
1125
|
+
end
|
1126
|
+
|
1127
|
+
def test_yylex_or2_equals
|
1128
|
+
util_lex_token "||=", :tOP_ASGN, "||"
|
1129
|
+
end
|
1130
|
+
|
1131
|
+
def test_yylex_or_equals
|
1132
|
+
util_lex_token "|=", :tOP_ASGN, "|"
|
1133
|
+
end
|
1134
|
+
|
1135
|
+
def test_yylex_percent
|
1136
|
+
util_lex_token("a % 2",
|
1137
|
+
:tIDENTIFIER, "a",
|
1138
|
+
:tPERCENT, "%",
|
1139
|
+
:tINTEGER, 2)
|
1140
|
+
end
|
1141
|
+
|
1142
|
+
def test_yylex_percent_equals
|
1143
|
+
util_lex_token("a %= 2",
|
1144
|
+
:tIDENTIFIER, "a",
|
1145
|
+
:tOP_ASGN, "%",
|
1146
|
+
:tINTEGER, 2)
|
1147
|
+
end
|
1148
|
+
|
1149
|
+
def test_yylex_plus
|
1150
|
+
util_lex_token("1 + 1", # TODO state?
|
1151
|
+
:tINTEGER, 1,
|
1152
|
+
:tPLUS, "+",
|
1153
|
+
:tINTEGER, 1)
|
1154
|
+
end
|
1155
|
+
|
1156
|
+
def test_yylex_plus_equals
|
1157
|
+
@lex.state = :expr_end
|
1158
|
+
|
1159
|
+
util_lex_token "+=", :tOP_ASGN, "+"
|
1160
|
+
end
|
1161
|
+
|
1162
|
+
def test_yylex_plus_method
|
1163
|
+
@lex.state = :expr_fname
|
1164
|
+
util_lex_token "+", :tPLUS, "+"
|
1165
|
+
end
|
1166
|
+
|
1167
|
+
def test_yylex_plus_unary_method
|
1168
|
+
@lex.state = :expr_fname
|
1169
|
+
util_lex_token "+@", :tUPLUS, "+@"
|
1170
|
+
end
|
1171
|
+
|
1172
|
+
def test_yylex_numbers
|
1173
|
+
util_lex_token "0b10", :tINTEGER, 2
|
1174
|
+
util_lex_token "0B10", :tINTEGER, 2
|
1175
|
+
|
1176
|
+
util_lex_token "0d10", :tINTEGER, 10
|
1177
|
+
util_lex_token "0D10", :tINTEGER, 10
|
1178
|
+
|
1179
|
+
util_lex_token "0x10", :tINTEGER, 16
|
1180
|
+
util_lex_token "0X10", :tINTEGER, 16
|
1181
|
+
|
1182
|
+
util_lex_token "0o10", :tINTEGER, 8
|
1183
|
+
util_lex_token "0O10", :tINTEGER, 8
|
1184
|
+
util_lex_token "0o", :tINTEGER, 0
|
1185
|
+
util_lex_token "0O", :tINTEGER, 0
|
1186
|
+
|
1187
|
+
util_lex_token "0o", :tINTEGER, 0
|
1188
|
+
util_lex_token "0O", :tINTEGER, 0
|
1189
|
+
|
1190
|
+
util_lex_token "0", :tINTEGER, 0
|
1191
|
+
|
1192
|
+
util_bad_token "0x"
|
1193
|
+
util_bad_token "0X"
|
1194
|
+
util_bad_token "0b"
|
1195
|
+
util_bad_token "0B"
|
1196
|
+
util_bad_token "0d"
|
1197
|
+
util_bad_token "0D"
|
1198
|
+
|
1199
|
+
util_bad_token "08"
|
1200
|
+
util_bad_token "09"
|
1201
|
+
util_bad_token "0o8"
|
1202
|
+
util_bad_token "0o9"
|
1203
|
+
util_bad_token "0O8"
|
1204
|
+
util_bad_token "0O9"
|
1205
|
+
|
1206
|
+
util_bad_token "1_e1"
|
1207
|
+
util_bad_token "1_.1"
|
1208
|
+
util_bad_token "1__1"
|
1209
|
+
end
|
1210
|
+
|
1211
|
+
def test_yylex_plus_unary_number
|
1212
|
+
util_lex_token("+42",
|
1213
|
+
:tINTEGER, 42)
|
1214
|
+
end
|
1215
|
+
|
1216
|
+
def test_yylex_question__18
|
1217
|
+
@lex = RubyLexer.new 18
|
1218
|
+
|
1219
|
+
util_lex_token "?*", :tINTEGER, 42
|
1220
|
+
end
|
1221
|
+
|
1222
|
+
def test_yylex_question__19
|
1223
|
+
@lex = RubyLexer.new 19
|
1224
|
+
|
1225
|
+
util_lex_token "?*", :tSTRING, "*"
|
1226
|
+
end
|
1227
|
+
|
1228
|
+
def test_yylex_question_bad_eos
|
1229
|
+
util_bad_token "?"
|
1230
|
+
end
|
1231
|
+
|
1232
|
+
def test_yylex_question_bad_ws
|
1233
|
+
util_lex_token "? ", :tEH, "?"
|
1234
|
+
util_lex_token "?\n", :tEH, "?"
|
1235
|
+
util_lex_token "?\t", :tEH, "?"
|
1236
|
+
util_lex_token "?\v", :tEH, "?"
|
1237
|
+
util_lex_token "?\r", :tEH, "?"
|
1238
|
+
util_lex_token "?\f", :tEH, "?"
|
1239
|
+
end
|
1240
|
+
|
1241
|
+
def test_yylex_question_ws_backslashed__18
|
1242
|
+
@lex = RubyLexer.new 18
|
1243
|
+
|
1244
|
+
@lex.state = :expr_beg
|
1245
|
+
util_lex_token "?\\ ", :tINTEGER, 32
|
1246
|
+
@lex.state = :expr_beg
|
1247
|
+
util_lex_token "?\\n", :tINTEGER, 10
|
1248
|
+
@lex.state = :expr_beg
|
1249
|
+
util_lex_token "?\\t", :tINTEGER, 9
|
1250
|
+
@lex.state = :expr_beg
|
1251
|
+
util_lex_token "?\\v", :tINTEGER, 11
|
1252
|
+
@lex.state = :expr_beg
|
1253
|
+
util_lex_token "?\\r", :tINTEGER, 13
|
1254
|
+
@lex.state = :expr_beg
|
1255
|
+
util_lex_token "?\\f", :tINTEGER, 12
|
1256
|
+
end
|
1257
|
+
|
1258
|
+
def test_yylex_question_ws_backslashed__19
|
1259
|
+
@lex = RubyLexer.new 19
|
1260
|
+
|
1261
|
+
@lex.state = :expr_beg
|
1262
|
+
util_lex_token "?\\ ", :tSTRING, " "
|
1263
|
+
@lex.state = :expr_beg
|
1264
|
+
util_lex_token "?\\n", :tSTRING, "\n"
|
1265
|
+
@lex.state = :expr_beg
|
1266
|
+
util_lex_token "?\\t", :tSTRING, "\t"
|
1267
|
+
@lex.state = :expr_beg
|
1268
|
+
util_lex_token "?\\v", :tSTRING, "\v"
|
1269
|
+
@lex.state = :expr_beg
|
1270
|
+
util_lex_token "?\\r", :tSTRING, "\r"
|
1271
|
+
@lex.state = :expr_beg
|
1272
|
+
util_lex_token "?\\f", :tSTRING, "\f"
|
1273
|
+
end
|
1274
|
+
|
1275
|
+
def test_yylex_rbracket
|
1276
|
+
util_lex_token "]", :tRBRACK, "]"
|
1277
|
+
end
|
1278
|
+
|
1279
|
+
def test_yylex_rcurly
|
1280
|
+
util_lex_token "}", :tRCURLY, "}"
|
1281
|
+
end
|
1282
|
+
|
1283
|
+
def test_yylex_regexp
|
1284
|
+
util_lex_token("/regexp/",
|
1285
|
+
:tREGEXP_BEG, "/",
|
1286
|
+
:tSTRING_CONTENT, "regexp",
|
1287
|
+
:tSTRING_END, "/",
|
1288
|
+
:tREGEXP_OPT, "")
|
1289
|
+
end
|
1290
|
+
|
1291
|
+
def test_yylex_regexp_ambiguous
|
1292
|
+
util_lex_token("method /regexp/",
|
1293
|
+
:tIDENTIFIER, "method",
|
1294
|
+
:tREGEXP_BEG, "/",
|
1295
|
+
:tSTRING_CONTENT, "regexp",
|
1296
|
+
:tSTRING_END, "/",
|
1297
|
+
:tREGEXP_OPT, "")
|
1298
|
+
end
|
1299
|
+
|
1300
|
+
def test_yylex_regexp_bad
|
1301
|
+
util_bad_token("/.*/xyz",
|
1302
|
+
:tREGEXP_BEG, "/",
|
1303
|
+
:tSTRING_CONTENT, ".*")
|
1304
|
+
end
|
1305
|
+
|
1306
|
+
def test_yylex_regexp_escape_C
|
1307
|
+
util_lex_token('/regex\\C-x/',
|
1308
|
+
:tREGEXP_BEG, "/",
|
1309
|
+
:tSTRING_CONTENT, "regex\\C-x",
|
1310
|
+
:tSTRING_END, "/",
|
1311
|
+
:tREGEXP_OPT, "")
|
1312
|
+
end
|
1313
|
+
|
1314
|
+
def test_yylex_regexp_escape_C_M
|
1315
|
+
util_lex_token('/regex\\C-\\M-x/',
|
1316
|
+
:tREGEXP_BEG, "/",
|
1317
|
+
:tSTRING_CONTENT, "regex\\C-\\M-x",
|
1318
|
+
:tSTRING_END, "/",
|
1319
|
+
:tREGEXP_OPT, "")
|
1320
|
+
end
|
1321
|
+
|
1322
|
+
def test_yylex_regexp_escape_C_M_craaaazy
|
1323
|
+
util_lex_token("/regex\\C-\\\n\\M-x/",
|
1324
|
+
:tREGEXP_BEG, "/",
|
1325
|
+
:tSTRING_CONTENT, "regex\\C-\\M-x",
|
1326
|
+
:tSTRING_END, "/",
|
1327
|
+
:tREGEXP_OPT, "")
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
def test_yylex_regexp_escape_C_bad_dash
|
1331
|
+
util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
|
1332
|
+
end
|
1333
|
+
|
1334
|
+
def test_yylex_regexp_escape_C_bad_dash_eos
|
1335
|
+
util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
|
1336
|
+
end
|
1337
|
+
|
1338
|
+
def test_yylex_regexp_escape_C_bad_dash_eos2
|
1339
|
+
util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
|
1340
|
+
end
|
1341
|
+
|
1342
|
+
def test_yylex_regexp_escape_C_bad_eos
|
1343
|
+
util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
|
1344
|
+
end
|
1345
|
+
|
1346
|
+
def test_yylex_regexp_escape_C_bad_eos2
|
1347
|
+
util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
|
1348
|
+
end
|
1349
|
+
|
1350
|
+
def test_yylex_regexp_escape_M
|
1351
|
+
util_lex_token('/regex\\M-x/',
|
1352
|
+
:tREGEXP_BEG, "/",
|
1353
|
+
:tSTRING_CONTENT, "regex\\M-x",
|
1354
|
+
:tSTRING_END, "/",
|
1355
|
+
:tREGEXP_OPT, "")
|
1356
|
+
end
|
1357
|
+
|
1358
|
+
def test_yylex_regexp_escape_M_C
|
1359
|
+
util_lex_token('/regex\\M-\\C-x/',
|
1360
|
+
:tREGEXP_BEG, "/",
|
1361
|
+
:tSTRING_CONTENT, "regex\\M-\\C-x",
|
1362
|
+
:tSTRING_END, "/",
|
1363
|
+
:tREGEXP_OPT, "")
|
1364
|
+
end
|
1365
|
+
|
1366
|
+
def test_yylex_regexp_escape_M_bad_dash
|
1367
|
+
util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
|
1368
|
+
end
|
1369
|
+
|
1370
|
+
def test_yylex_regexp_escape_M_bad_dash_eos
|
1371
|
+
util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
|
1372
|
+
end
|
1373
|
+
|
1374
|
+
def test_yylex_regexp_escape_M_bad_dash_eos2
|
1375
|
+
util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
|
1376
|
+
end
|
1377
|
+
|
1378
|
+
def test_yylex_regexp_escape_M_bad_eos
|
1379
|
+
util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
|
1380
|
+
end
|
1381
|
+
|
1382
|
+
def test_yylex_regexp_escape_backslash_slash
|
1383
|
+
util_lex_token('/\\//',
|
1384
|
+
:tREGEXP_BEG, "/",
|
1385
|
+
:tSTRING_CONTENT, '\\/',
|
1386
|
+
:tSTRING_END, "/",
|
1387
|
+
:tREGEXP_OPT, "")
|
1388
|
+
end
|
1389
|
+
|
1390
|
+
def test_yylex_regexp_escape_backslash_terminator
|
1391
|
+
util_lex_token('%r%blah\\%blah%',
|
1392
|
+
:tREGEXP_BEG, "%r",
|
1393
|
+
:tSTRING_CONTENT, "blah\\%blah",
|
1394
|
+
:tSTRING_END, "%",
|
1395
|
+
:tREGEXP_OPT, "")
|
1396
|
+
end
|
1397
|
+
|
1398
|
+
def test_yylex_regexp_escape_backslash_terminator_meta1
|
1399
|
+
util_lex_token('%r{blah\\}blah}',
|
1400
|
+
:tREGEXP_BEG, "%r",
|
1401
|
+
:tSTRING_CONTENT, "blah\\}blah",
|
1402
|
+
:tSTRING_END, "}",
|
1403
|
+
:tREGEXP_OPT, "")
|
1404
|
+
end
|
1405
|
+
|
1406
|
+
def test_yylex_regexp_escape_backslash_terminator_meta2
|
1407
|
+
util_lex_token('%r/blah\\/blah/',
|
1408
|
+
:tREGEXP_BEG, "%r",
|
1409
|
+
:tSTRING_CONTENT, "blah\\/blah",
|
1410
|
+
:tSTRING_END, "/",
|
1411
|
+
:tREGEXP_OPT, "")
|
1412
|
+
end
|
1413
|
+
|
1414
|
+
def test_yylex_regexp_escape_backslash_terminator_meta3
|
1415
|
+
util_lex_token('%r/blah\\%blah/',
|
1416
|
+
:tREGEXP_BEG, "%r",
|
1417
|
+
:tSTRING_CONTENT, "blah\\%blah",
|
1418
|
+
:tSTRING_END, "/",
|
1419
|
+
:tREGEXP_OPT, "")
|
1420
|
+
end
|
1421
|
+
|
1422
|
+
def test_yylex_regexp_escape_bad_eos
|
1423
|
+
util_bad_token '/regex\\', :tREGEXP_BEG, "/"
|
1424
|
+
end
|
1425
|
+
|
1426
|
+
def test_yylex_regexp_escape_bs
|
1427
|
+
util_lex_token('/regex\\\\regex/',
|
1428
|
+
:tREGEXP_BEG, "/",
|
1429
|
+
:tSTRING_CONTENT, "regex\\\\regex",
|
1430
|
+
:tSTRING_END, "/",
|
1431
|
+
:tREGEXP_OPT, "")
|
1432
|
+
end
|
1433
|
+
|
1434
|
+
def test_yylex_regexp_escape_c
|
1435
|
+
util_lex_token('/regex\\cxxx/',
|
1436
|
+
:tREGEXP_BEG, "/",
|
1437
|
+
:tSTRING_CONTENT, "regex\\cxxx",
|
1438
|
+
:tSTRING_END, "/",
|
1439
|
+
:tREGEXP_OPT, "")
|
1440
|
+
end
|
1441
|
+
|
1442
|
+
def test_yylex_regexp_escape_c_backslash
|
1443
|
+
util_lex_token('/regex\\c\\n/',
|
1444
|
+
:tREGEXP_BEG, "/",
|
1445
|
+
:tSTRING_CONTENT, "regex\\c\\n",
|
1446
|
+
:tSTRING_END, "/",
|
1447
|
+
:tREGEXP_OPT, "")
|
1448
|
+
end
|
1449
|
+
|
1450
|
+
def test_yylex_regexp_escape_chars
|
1451
|
+
util_lex_token('/re\\tge\\nxp/',
|
1452
|
+
:tREGEXP_BEG, "/",
|
1453
|
+
:tSTRING_CONTENT, "re\\tge\\nxp",
|
1454
|
+
:tSTRING_END, "/",
|
1455
|
+
:tREGEXP_OPT, "")
|
1456
|
+
end
|
1457
|
+
|
1458
|
+
def test_yylex_regexp_escape_double_backslash
|
1459
|
+
regexp = '/[\\/\\\\]$/'
|
1460
|
+
util_lex_token(regexp,
|
1461
|
+
:tREGEXP_BEG, "/",
|
1462
|
+
:tSTRING_CONTENT, regexp[1..-2],
|
1463
|
+
:tSTRING_END, "/",
|
1464
|
+
:tREGEXP_OPT, "")
|
1465
|
+
end
|
1466
|
+
|
1467
|
+
def test_yylex_regexp_escape_hex
|
1468
|
+
util_lex_token('/regex\\x61xp/',
|
1469
|
+
:tREGEXP_BEG, "/",
|
1470
|
+
:tSTRING_CONTENT, "regex\\x61xp",
|
1471
|
+
:tSTRING_END, "/",
|
1472
|
+
:tREGEXP_OPT, "")
|
1473
|
+
end
|
1474
|
+
|
1475
|
+
def test_yylex_regexp_escape_hex_bad
|
1476
|
+
util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
|
1477
|
+
end
|
1478
|
+
|
1479
|
+
def test_yylex_regexp_escape_hex_one
|
1480
|
+
util_lex_token('/^[\\xd\\xa]{2}/on',
|
1481
|
+
:tREGEXP_BEG, '/',
|
1482
|
+
:tSTRING_CONTENT, '^[\\xd\\xa]{2}',
|
1483
|
+
:tSTRING_END, "/",
|
1484
|
+
:tREGEXP_OPT, 'on')
|
1485
|
+
end
|
1486
|
+
|
1487
|
+
def test_yylex_regexp_escape_oct1
|
1488
|
+
util_lex_token('/regex\\0xp/',
|
1489
|
+
:tREGEXP_BEG, "/",
|
1490
|
+
:tSTRING_CONTENT, "regex\\0xp",
|
1491
|
+
:tSTRING_END, "/",
|
1492
|
+
:tREGEXP_OPT, "")
|
1493
|
+
end
|
1494
|
+
|
1495
|
+
def test_yylex_regexp_escape_oct2
|
1496
|
+
util_lex_token('/regex\\07xp/',
|
1497
|
+
:tREGEXP_BEG, "/",
|
1498
|
+
:tSTRING_CONTENT, "regex\\07xp",
|
1499
|
+
:tSTRING_END, "/",
|
1500
|
+
:tREGEXP_OPT, "")
|
1501
|
+
end
|
1502
|
+
|
1503
|
+
def test_yylex_regexp_escape_oct3
|
1504
|
+
util_lex_token('/regex\\10142/',
|
1505
|
+
:tREGEXP_BEG, "/",
|
1506
|
+
:tSTRING_CONTENT, "regex\\10142",
|
1507
|
+
:tSTRING_END, "/",
|
1508
|
+
:tREGEXP_OPT, "")
|
1509
|
+
end
|
1510
|
+
|
1511
|
+
def test_yylex_regexp_escape_return
|
1512
|
+
util_lex_token("/regex\\\nregex/",
|
1513
|
+
:tREGEXP_BEG, "/",
|
1514
|
+
:tSTRING_CONTENT, "regexregex",
|
1515
|
+
:tSTRING_END, "/",
|
1516
|
+
:tREGEXP_OPT, "")
|
1517
|
+
end
|
1518
|
+
|
1519
|
+
def test_yylex_regexp_nm
|
1520
|
+
util_lex_token("/.*/nm",
|
1521
|
+
:tREGEXP_BEG, "/",
|
1522
|
+
:tSTRING_CONTENT, ".*",
|
1523
|
+
:tSTRING_END, "/",
|
1524
|
+
:tREGEXP_OPT, "nm")
|
1525
|
+
end
|
1526
|
+
|
1527
|
+
def test_yylex_rparen
|
1528
|
+
util_lex_token ")", :tRPAREN, ")"
|
1529
|
+
end
|
1530
|
+
|
1531
|
+
def test_yylex_rshft
|
1532
|
+
util_lex_token("a >> 2",
|
1533
|
+
:tIDENTIFIER, "a",
|
1534
|
+
:tRSHFT, ">>",
|
1535
|
+
:tINTEGER, 2)
|
1536
|
+
end
|
1537
|
+
|
1538
|
+
def test_yylex_rshft_equals
|
1539
|
+
util_lex_token("a >>= 2",
|
1540
|
+
:tIDENTIFIER, "a",
|
1541
|
+
:tOP_ASGN, ">>",
|
1542
|
+
:tINTEGER, 2)
|
1543
|
+
end
|
1544
|
+
|
1545
|
+
def test_yylex_star
|
1546
|
+
util_lex_token("a * ",
|
1547
|
+
:tIDENTIFIER, "a",
|
1548
|
+
:tSTAR2, "*")
|
1549
|
+
|
1550
|
+
assert_equal :expr_beg, @lex.state
|
1551
|
+
end
|
1552
|
+
|
1553
|
+
def test_yylex_star2
|
1554
|
+
util_lex_token("a ** ",
|
1555
|
+
:tIDENTIFIER, "a",
|
1556
|
+
:tPOW, "**")
|
1557
|
+
|
1558
|
+
assert_equal :expr_beg, @lex.state
|
1559
|
+
end
|
1560
|
+
|
1561
|
+
def test_yylex_star2_equals
|
1562
|
+
util_lex_token("a **= ",
|
1563
|
+
:tIDENTIFIER, "a",
|
1564
|
+
:tOP_ASGN, "**")
|
1565
|
+
|
1566
|
+
assert_equal :expr_beg, @lex.state
|
1567
|
+
end
|
1568
|
+
|
1569
|
+
def test_yylex_star_arg
|
1570
|
+
@lex.state = :expr_arg
|
1571
|
+
|
1572
|
+
util_lex_token(" *a",
|
1573
|
+
:tSTAR, "*",
|
1574
|
+
:tIDENTIFIER, "a")
|
1575
|
+
|
1576
|
+
assert_equal :expr_arg, @lex.state
|
1577
|
+
end
|
1578
|
+
|
1579
|
+
def test_yylex_star_arg_beg
|
1580
|
+
@lex.state = :expr_beg
|
1581
|
+
|
1582
|
+
util_lex_token("*a",
|
1583
|
+
:tSTAR, "*",
|
1584
|
+
:tIDENTIFIER, "a")
|
1585
|
+
|
1586
|
+
assert_equal :expr_arg, @lex.state
|
1587
|
+
end
|
1588
|
+
|
1589
|
+
def test_yylex_star_arg_beg_fname
|
1590
|
+
@lex.state = :expr_fname
|
1591
|
+
|
1592
|
+
util_lex_token("*a",
|
1593
|
+
:tSTAR2, "*",
|
1594
|
+
:tIDENTIFIER, "a")
|
1595
|
+
|
1596
|
+
assert_equal :expr_arg, @lex.state
|
1597
|
+
end
|
1598
|
+
|
1599
|
+
def test_yylex_star_equals
|
1600
|
+
util_lex_token("a *= ",
|
1601
|
+
:tIDENTIFIER, "a",
|
1602
|
+
:tOP_ASGN, "*")
|
1603
|
+
|
1604
|
+
assert_equal :expr_beg, @lex.state
|
1605
|
+
end
|
1606
|
+
|
1607
|
+
def test_yylex_string_bad_eos
|
1608
|
+
util_bad_token('%',
|
1609
|
+
:tSTRING_BEG, '%')
|
1610
|
+
end
|
1611
|
+
|
1612
|
+
def test_yylex_string_bad_eos_quote
|
1613
|
+
util_bad_token('%{nest',
|
1614
|
+
:tSTRING_BEG, '%}')
|
1615
|
+
end
|
1616
|
+
|
1617
|
+
def test_yylex_string_double
|
1618
|
+
util_lex_token('"string"',
|
1619
|
+
:tSTRING, "string")
|
1620
|
+
end
|
1621
|
+
|
1622
|
+
def test_yylex_string_double_escape_C
|
1623
|
+
util_lex_token('"\\C-a"',
|
1624
|
+
:tSTRING, "\001")
|
1625
|
+
end
|
1626
|
+
|
1627
|
+
def test_yylex_string_double_escape_C_backslash
|
1628
|
+
util_lex_token('"\\C-\\\\"',
|
1629
|
+
:tSTRING, "\034")
|
1630
|
+
end
|
1631
|
+
|
1632
|
+
def test_yylex_string_double_escape_C_escape
|
1633
|
+
util_lex_token('"\\C-\\M-a"',
|
1634
|
+
:tSTRING, "\201")
|
1635
|
+
end
|
1636
|
+
|
1637
|
+
def test_yylex_string_double_escape_C_question
|
1638
|
+
util_lex_token('"\\C-?"',
|
1639
|
+
:tSTRING, "\177")
|
1640
|
+
end
|
1641
|
+
|
1642
|
+
def test_yylex_string_double_escape_M
|
1643
|
+
util_lex_token('"\\M-a"',
|
1644
|
+
:tSTRING, "\341")
|
1645
|
+
end
|
1646
|
+
|
1647
|
+
def test_yylex_string_double_escape_M_backslash
|
1648
|
+
util_lex_token('"\\M-\\\\"',
|
1649
|
+
:tSTRING, "\334")
|
1650
|
+
end
|
1651
|
+
|
1652
|
+
def test_yylex_string_double_escape_M_escape
|
1653
|
+
util_lex_token('"\\M-\\C-a"',
|
1654
|
+
:tSTRING, "\201")
|
1655
|
+
end
|
1656
|
+
|
1657
|
+
def test_yylex_string_double_escape_bs1
|
1658
|
+
util_lex_token('"a\\a\\a"',
|
1659
|
+
:tSTRING, "a\a\a")
|
1660
|
+
end
|
1661
|
+
|
1662
|
+
def test_yylex_string_double_escape_bs2
|
1663
|
+
util_lex_token('"a\\\\a"',
|
1664
|
+
:tSTRING, "a\\a")
|
1665
|
+
end
|
1666
|
+
|
1667
|
+
def test_yylex_string_double_escape_c
|
1668
|
+
util_lex_token('"\\ca"',
|
1669
|
+
:tSTRING, "\001")
|
1670
|
+
end
|
1671
|
+
|
1672
|
+
def test_yylex_string_double_escape_c_escape
|
1673
|
+
util_lex_token('"\\c\\M-a"',
|
1674
|
+
:tSTRING, "\201")
|
1675
|
+
end
|
1676
|
+
|
1677
|
+
def test_yylex_string_double_escape_c_question
|
1678
|
+
util_lex_token('"\\c?"',
|
1679
|
+
:tSTRING, "\177")
|
1680
|
+
end
|
1681
|
+
|
1682
|
+
def test_yylex_string_double_escape_chars
|
1683
|
+
util_lex_token('"s\\tri\\ng"',
|
1684
|
+
:tSTRING, "s\tri\ng")
|
1685
|
+
end
|
1686
|
+
|
1687
|
+
def test_yylex_string_double_escape_hex
|
1688
|
+
util_lex_token('"n = \\x61\\x62\\x63"',
|
1689
|
+
:tSTRING, "n = abc")
|
1690
|
+
end
|
1691
|
+
|
1692
|
+
def test_yylex_string_double_escape_octal
|
1693
|
+
util_lex_token('"n = \\101\\102\\103"',
|
1694
|
+
:tSTRING, "n = ABC")
|
1695
|
+
end
|
1696
|
+
|
1697
|
+
def test_yylex_string_double_interp
|
1698
|
+
util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
|
1699
|
+
:tSTRING_BEG, "\"",
|
1700
|
+
:tSTRING_CONTENT, "blah #x a ",
|
1701
|
+
:tSTRING_DVAR, nil,
|
1702
|
+
:tIVAR, "@a",
|
1703
|
+
:tSTRING_CONTENT, " b ",
|
1704
|
+
:tSTRING_DVAR, nil,
|
1705
|
+
:tGVAR, "$b",
|
1706
|
+
:tSTRING_CONTENT, " c ",
|
1707
|
+
:tSTRING_DBEG, '#{',
|
1708
|
+
:tINTEGER, 3,
|
1709
|
+
:tRCURLY, "}",
|
1710
|
+
:tSTRING_CONTENT, " # ",
|
1711
|
+
:tSTRING_END, "\"")
|
1712
|
+
end
|
1713
|
+
|
1714
|
+
def test_yylex_string_double_nested_curlies
|
1715
|
+
util_lex_token('%{nest{one{two}one}nest}',
|
1716
|
+
:tSTRING, "nest{one{two}one}nest")
|
1717
|
+
end
|
1718
|
+
|
1719
|
+
def test_yylex_string_double_no_interp
|
1720
|
+
util_lex_token("\"# blah\"", # pound first
|
1721
|
+
:tSTRING, "# blah")
|
1722
|
+
|
1723
|
+
util_lex_token("\"blah # blah\"", # pound not first
|
1724
|
+
:tSTRING, "blah # blah")
|
1725
|
+
end
|
1726
|
+
|
1727
|
+
def test_yylex_string_escape_x_single
|
1728
|
+
util_lex_token('"\\x0"',
|
1729
|
+
:tSTRING, "\000")
|
1730
|
+
end
|
1731
|
+
|
1732
|
+
def test_yylex_string_pct_Q
|
1733
|
+
util_lex_token("%Q[s1 s2]",
|
1734
|
+
:tSTRING, "s1 s2")
|
1735
|
+
end
|
1736
|
+
|
1737
|
+
def test_yylex_string_pct_W
|
1738
|
+
util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
|
1739
|
+
:tWORDS_BEG, "%W",
|
1740
|
+
:tSTRING_CONTENT, "s1",
|
1741
|
+
:tSPACE, nil,
|
1742
|
+
:tSTRING_CONTENT, "s2",
|
1743
|
+
:tSPACE, nil,
|
1744
|
+
:tSTRING_CONTENT, "s3",
|
1745
|
+
:tSPACE, nil,
|
1746
|
+
:tSTRING_END, ']')
|
1747
|
+
end
|
1748
|
+
|
1749
|
+
def test_yylex_string_pct_W_bs_nl
|
1750
|
+
util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
|
1751
|
+
:tWORDS_BEG, "%W",
|
1752
|
+
:tSTRING_CONTENT, "s1",
|
1753
|
+
:tSPACE, nil,
|
1754
|
+
:tSTRING_CONTENT, "\ns2",
|
1755
|
+
:tSPACE, nil,
|
1756
|
+
:tSTRING_END, ']')
|
1757
|
+
end
|
1758
|
+
|
1759
|
+
def test_yylex_string_pct_angle
|
1760
|
+
util_lex_token("%<blah>",
|
1761
|
+
:tSTRING, "blah")
|
1762
|
+
end
|
1763
|
+
|
1764
|
+
def test_yylex_string_pct_other
|
1765
|
+
util_lex_token("%%blah%",
|
1766
|
+
:tSTRING, "blah")
|
1767
|
+
end
|
1768
|
+
|
1769
|
+
def test_yylex_string_pct_w
|
1770
|
+
util_bad_token("%w[s1 s2 ",
|
1771
|
+
:tQWORDS_BEG, "%w",
|
1772
|
+
:tSTRING_CONTENT, "s1",
|
1773
|
+
:tSPACE, nil,
|
1774
|
+
:tSTRING_CONTENT, "s2",
|
1775
|
+
:tSPACE, nil)
|
1776
|
+
end
|
1777
|
+
|
1778
|
+
def test_yylex_string_pct_w_bs_nl
|
1779
|
+
util_lex_token("%w[s1 \\\ns2]",
|
1780
|
+
:tQWORDS_BEG, "%w",
|
1781
|
+
:tSTRING_CONTENT, "s1",
|
1782
|
+
:tSPACE, nil,
|
1783
|
+
:tSTRING_CONTENT, "\ns2",
|
1784
|
+
:tSPACE, nil,
|
1785
|
+
:tSTRING_END, ']')
|
1786
|
+
end
|
1787
|
+
|
1788
|
+
def test_yylex_string_pct_w_bs_sp
|
1789
|
+
util_lex_token("%w[s\\ 1 s\\ 2]",
|
1790
|
+
:tQWORDS_BEG, "%w",
|
1791
|
+
:tSTRING_CONTENT, "s 1",
|
1792
|
+
:tSPACE, nil,
|
1793
|
+
:tSTRING_CONTENT, "s 2",
|
1794
|
+
:tSPACE, nil,
|
1795
|
+
:tSTRING_END, ']')
|
1796
|
+
end
|
1797
|
+
|
1798
|
+
def test_yylex_string_pct_w_tab
|
1799
|
+
util_lex_token("%w[abc\tdef]",
|
1800
|
+
:tQWORDS_BEG, "%w",
|
1801
|
+
:tSTRING_CONTENT, "abc",
|
1802
|
+
:tSPACE, nil,
|
1803
|
+
:tSTRING_CONTENT, "def",
|
1804
|
+
:tSPACE, nil,
|
1805
|
+
:tSTRING_END, ']')
|
1806
|
+
end
|
1807
|
+
|
1808
|
+
def test_yylex_string_single
|
1809
|
+
util_lex_token("'string'",
|
1810
|
+
:tSTRING, "string")
|
1811
|
+
end
|
1812
|
+
|
1813
|
+
def test_yylex_string_single_escape_chars
|
1814
|
+
util_lex_token("'s\\tri\\ng'",
|
1815
|
+
:tSTRING, "s\\tri\\ng")
|
1816
|
+
end
|
1817
|
+
|
1818
|
+
def test_yylex_string_single_nl
|
1819
|
+
util_lex_token("'blah\\\nblah'",
|
1820
|
+
:tSTRING, "blah\\\nblah")
|
1821
|
+
end
|
1822
|
+
|
1823
|
+
def test_yylex_symbol
|
1824
|
+
util_lex_token(":symbol",
|
1825
|
+
:tSYMBOL, "symbol")
|
1826
|
+
end
|
1827
|
+
|
1828
|
+
def test_yylex_symbol_bad_zero
|
1829
|
+
util_bad_token(":\"blah\0\"",
|
1830
|
+
:tSYMBEG, ":")
|
1831
|
+
end
|
1832
|
+
|
1833
|
+
def test_yylex_symbol_double
|
1834
|
+
util_lex_token(":\"symbol\"",
|
1835
|
+
:tSYMBOL, "symbol")
|
1836
|
+
end
|
1837
|
+
|
1838
|
+
def test_yylex_symbol_single
|
1839
|
+
util_lex_token(":'symbol'",
|
1840
|
+
:tSYMBOL, "symbol")
|
1841
|
+
end
|
1842
|
+
|
1843
|
+
def test_yylex_ternary
|
1844
|
+
util_lex_token("a ? b : c",
|
1845
|
+
:tIDENTIFIER, "a",
|
1846
|
+
:tEH, "?",
|
1847
|
+
:tIDENTIFIER, "b",
|
1848
|
+
:tCOLON, ":",
|
1849
|
+
:tIDENTIFIER, "c")
|
1850
|
+
|
1851
|
+
util_lex_token("a ?b : c",
|
1852
|
+
:tIDENTIFIER, "a",
|
1853
|
+
:tINTEGER, 98,
|
1854
|
+
:tCOLON, ":",
|
1855
|
+
:tIDENTIFIER, "c")
|
1856
|
+
|
1857
|
+
util_lex_token("a ?bb : c", # GAH! MATZ!!!
|
1858
|
+
:tIDENTIFIER, "a",
|
1859
|
+
:tEH, "?",
|
1860
|
+
:tIDENTIFIER, "bb",
|
1861
|
+
:tCOLON, ":",
|
1862
|
+
:tIDENTIFIER, "c")
|
1863
|
+
|
1864
|
+
util_lex_token("42 ?", # 42 forces expr_end
|
1865
|
+
:tINTEGER, 42,
|
1866
|
+
:tEH, "?")
|
1867
|
+
end
|
1868
|
+
|
1869
|
+
def test_yylex_tilde
|
1870
|
+
util_lex_token "~", :tTILDE, "~"
|
1871
|
+
end
|
1872
|
+
|
1873
|
+
def test_yylex_tilde_unary
|
1874
|
+
@lex.state = :expr_fname
|
1875
|
+
util_lex_token "~@", :tTILDE, "~@"
|
1876
|
+
end
|
1877
|
+
|
1878
|
+
def test_yylex_uminus
|
1879
|
+
util_lex_token("-blah",
|
1880
|
+
:tUMINUS, "-",
|
1881
|
+
:tIDENTIFIER, "blah")
|
1882
|
+
end
|
1883
|
+
|
1884
|
+
def test_yylex_underscore
|
1885
|
+
util_lex_token("_var", :tIDENTIFIER, "_var")
|
1886
|
+
end
|
1887
|
+
|
1888
|
+
def test_yylex_underscore_end
|
1889
|
+
@lex.source = "__END__\n"
|
1890
|
+
tok, = @lex.advance
|
1891
|
+
deny tok
|
1892
|
+
end
|
1893
|
+
|
1894
|
+
def test_yylex_uplus
|
1895
|
+
util_lex_token("+blah",
|
1896
|
+
:tUPLUS, "+",
|
1897
|
+
:tIDENTIFIER, "blah")
|
1898
|
+
end
|
1899
|
+
|
1900
|
+
def test_yylex_if_unless_mod
|
1901
|
+
util_lex_token("return if true unless false",
|
1902
|
+
:kRETURN, "return",
|
1903
|
+
:kIF_MOD, "if",
|
1904
|
+
:kTRUE, "true",
|
1905
|
+
:kUNLESS_MOD, "unless",
|
1906
|
+
:kFALSE, "false")
|
1907
|
+
end
|
1908
|
+
|
1909
|
+
def test_yylex_if_stmt
|
1910
|
+
util_lex_token("if true\n return end",
|
1911
|
+
:kIF, "if",
|
1912
|
+
:kTRUE, "true",
|
1913
|
+
:tNL, nil,
|
1914
|
+
:kRETURN, "return",
|
1915
|
+
:kEND, "end")
|
1916
|
+
end
|
1917
|
+
|
1918
|
+
def test_yylex_static_env
|
1919
|
+
env = RubyParserStuff::Environment.new
|
1920
|
+
env[:a] = :lvar
|
1921
|
+
@lex.static_env = env
|
1922
|
+
|
1923
|
+
util_lex_token("a [42]",
|
1924
|
+
:tIDENTIFIER, "a",
|
1925
|
+
:tLBRACK2, "[",
|
1926
|
+
:tINTEGER, 42,
|
1927
|
+
:tRBRACK, "]")
|
1928
|
+
end
|
1929
|
+
|
1930
|
+
def test_zbug_float_in_decl
|
1931
|
+
util_lex_token("def initialize(u = ",
|
1932
|
+
:kDEF, "def",
|
1933
|
+
:tIDENTIFIER, "initialize",
|
1934
|
+
:tLPAREN2, "(",
|
1935
|
+
:tIDENTIFIER, "u",
|
1936
|
+
:tEQL, "=")
|
1937
|
+
|
1938
|
+
assert_equal :expr_beg, @lex.state
|
1939
|
+
|
1940
|
+
util_lex_token("0.0, s = 0.0",
|
1941
|
+
:tFLOAT, 0.0,
|
1942
|
+
:tCOMMA, ',',
|
1943
|
+
:tIDENTIFIER, "s",
|
1944
|
+
:tEQL, "=",
|
1945
|
+
:tFLOAT, 0.0)
|
1946
|
+
end
|
1947
|
+
|
1948
|
+
def test_zbug_id_equals
|
1949
|
+
util_lex_token("a =",
|
1950
|
+
:tIDENTIFIER, "a",
|
1951
|
+
:tEQL, "=")
|
1952
|
+
|
1953
|
+
assert_equal :expr_beg, @lex.state
|
1954
|
+
|
1955
|
+
util_lex_token("0.0",
|
1956
|
+
:tFLOAT, 0.0)
|
1957
|
+
end
|
1958
|
+
|
1959
|
+
def test_zbug_no_spaces_in_decl
|
1960
|
+
util_lex_token("def initialize(u=",
|
1961
|
+
:kDEF, "def",
|
1962
|
+
:tIDENTIFIER, "initialize",
|
1963
|
+
:tLPAREN2, "(",
|
1964
|
+
:tIDENTIFIER, "u",
|
1965
|
+
:tEQL, "=")
|
1966
|
+
|
1967
|
+
assert_equal :expr_beg, @lex.state
|
1968
|
+
|
1969
|
+
util_lex_token("0.0,s=0.0",
|
1970
|
+
:tFLOAT, 0.0,
|
1971
|
+
:tCOMMA, ",",
|
1972
|
+
:tIDENTIFIER, "s",
|
1973
|
+
:tEQL, "=",
|
1974
|
+
:tFLOAT, 0.0)
|
1975
|
+
end
|
1976
|
+
|
1977
|
+
############################################################
|
1978
|
+
|
1979
|
+
def util_bad_token s, *args
|
1980
|
+
assert_raises RubyParser::SyntaxError do
|
1981
|
+
util_lex_token s, *args
|
1982
|
+
end
|
1983
|
+
end
|
1984
|
+
|
1985
|
+
def util_escape expected, input
|
1986
|
+
@lex.reset
|
1987
|
+
@lex.source = "%Q[\\#{input}]"
|
1988
|
+
|
1989
|
+
lex_token, lex_value = @lex.advance
|
1990
|
+
|
1991
|
+
if lex_value.respond_to?(:force_encoding)
|
1992
|
+
lex_value.force_encoding('ASCII-8BIT')
|
1993
|
+
end
|
1994
|
+
|
1995
|
+
assert_equal [:tSTRING, expected],
|
1996
|
+
[lex_token, lex_value],
|
1997
|
+
@lex.source
|
1998
|
+
end
|
1999
|
+
|
2000
|
+
def util_escape_bad input
|
2001
|
+
assert_raises RubyParser::SyntaxError do
|
2002
|
+
@lex.state = :expr_beg
|
2003
|
+
util_lex_token "%Q[\\#{input}]"
|
2004
|
+
end
|
2005
|
+
end
|
2006
|
+
|
2007
|
+
def util_lex_fname name, type, end_state = :expr_end
|
2008
|
+
util_lex_token("def #{name} ", :kDEF, "def", type, name)
|
2009
|
+
|
2010
|
+
assert_equal end_state, @lex.state
|
2011
|
+
end
|
2012
|
+
|
2013
|
+
def util_lex_token input, *args
|
2014
|
+
@lex.reset(false)
|
2015
|
+
@lex.source = input
|
2016
|
+
|
2017
|
+
until args.empty? do
|
2018
|
+
token, value = args.shift(2)
|
2019
|
+
|
2020
|
+
lex_token, lex_value = @lex.advance
|
2021
|
+
assert lex_token, "no more tokens"
|
2022
|
+
assert_equal [token, value], [lex_token, lex_value], input
|
2023
|
+
end
|
2024
|
+
|
2025
|
+
lex_token, lex_value = @lex.advance
|
2026
|
+
deny lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
|
2027
|
+
end
|
2028
|
+
end
|