parser 0.9.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2028 @@
1
+ #!/usr/bin/env ruby
2
+ # encoding: ascii-8bit
3
+
4
+ require 'minitest/autorun'
5
+ require 'ruby_lexer'
6
+ require 'ruby_parser_extras'
7
+
8
+ class TestRubyLexer < MiniTest::Unit::TestCase
9
+ alias :deny :refute
10
+
11
+ def setup
12
+ setup_lexer 18
13
+ end
14
+
15
+ def setup_lexer version
16
+ @lex = RubyLexer.new(version)
17
+ end
18
+
19
+ def test_advance
20
+ @lex.source = "blah"
21
+
22
+ token, = @lex.advance
23
+ assert token # blah
24
+
25
+ token, = @lex.advance
26
+ deny token # nada
27
+ end
28
+
29
+ def test_read_escape
30
+ util_escape "\\", "\\"
31
+ util_escape "\n", "n"
32
+ util_escape "\t", "t"
33
+ util_escape "\r", "r"
34
+ util_escape "\f", "f"
35
+ util_escape "\13", "v"
36
+ util_escape "\0", "0"
37
+ util_escape "\07", "a"
38
+ util_escape "\007", "a"
39
+ util_escape "\033", "e"
40
+ util_escape "\377", "377"
41
+ util_escape "\377", "xff"
42
+ util_escape "\010", "b"
43
+ util_escape " ", "s"
44
+ util_escape "q", "q" # plain vanilla escape
45
+ end
46
+
47
+ def test_read_escape_c
48
+ util_escape "\030", "C-x"
49
+ util_escape "\030", "cx"
50
+ util_escape "\230", 'C-\M-x'
51
+ util_escape "\230", 'c\M-x'
52
+
53
+ util_escape "\177", "C-?"
54
+ util_escape "\177", "c?"
55
+ end
56
+
57
+ def test_read_escape_m
58
+ util_escape "\370", "M-x"
59
+ util_escape "\230", 'M-\C-x'
60
+ util_escape "\230", 'M-\cx'
61
+ end
62
+
63
+ def test_read_escape_errors
64
+ util_escape_bad ""
65
+
66
+ util_escape_bad "M"
67
+ util_escape_bad "M-"
68
+ util_escape_bad "Mx"
69
+
70
+ util_escape_bad "Cx"
71
+ util_escape_bad "C"
72
+ util_escape_bad "C-"
73
+
74
+ util_escape_bad "c"
75
+ end
76
+
77
+ def test_read_escape_unicode__19
78
+ util_escape "\xc4\xa3", 'u0123'
79
+
80
+ util_escape "\xc4\xa3\xc3\xb0\xeb\x84\xa3", 'u{123 f0 B123}'
81
+ end
82
+
83
+ def test_read_escape_unicode_bad__19
84
+ util_escape_bad 'u123'
85
+ util_escape_bad 'u{}'
86
+ util_escape_bad 'u{123 f0h}'
87
+ util_escape_bad 'u{123 f0'
88
+ end
89
+
90
+ def test_yylex_ambiguous_uminus
91
+ util_lex_token("m -3",
92
+ :tIDENTIFIER, "m",
93
+ :tUMINUS_NUM, "-",
94
+ :tINTEGER, 3)
95
+ # TODO: verify warning
96
+ end
97
+
98
+ def test_yylex_ambiguous_uplus
99
+ util_lex_token("m +3",
100
+ :tIDENTIFIER, "m",
101
+ :tINTEGER, 3)
102
+ # TODO: verify warning
103
+ end
104
+
105
+ def test_yylex_and
106
+ util_lex_token "&", :tAMPER, "&"
107
+ end
108
+
109
+ def test_yylex_and2
110
+ @lex.state = :expr_end
111
+
112
+ util_lex_token "&&", :tANDOP, "&&"
113
+ end
114
+
115
+ def test_yylex_and2_equals
116
+ @lex.state = :expr_end
117
+
118
+ util_lex_token "&&=", :tOP_ASGN, "&&"
119
+ end
120
+
121
+ def test_yylex_and_arg
122
+ @lex.state = :expr_arg
123
+
124
+ util_lex_token(" &y",
125
+ :tAMPER, "&",
126
+ :tIDENTIFIER, "y")
127
+ end
128
+
129
+ def test_yylex_and_equals
130
+ @lex.state = :expr_end
131
+
132
+ util_lex_token "&=", :tOP_ASGN, "&"
133
+ end
134
+
135
+ def test_yylex_and_expr
136
+ @lex.state = :expr_arg
137
+
138
+ util_lex_token("x & y",
139
+ :tIDENTIFIER, "x",
140
+ :tAMPER2, "&",
141
+ :tIDENTIFIER, "y")
142
+ end
143
+
144
+ def test_yylex_and_meth
145
+ util_lex_fname "&", :tAMPER2
146
+ end
147
+
148
+ def test_yylex_assoc
149
+ util_lex_token "=>", :tASSOC, "=>"
150
+ end
151
+
152
+ def test_yylex_label__18
153
+ util_lex_token("{a:b",
154
+ :tLBRACE, "{",
155
+ :tIDENTIFIER, "a",
156
+ :tSYMBOL, "b")
157
+ end
158
+
159
+ def test_yylex_label_in_params__18
160
+ util_lex_token("foo(a:b",
161
+ :tIDENTIFIER, "foo",
162
+ :tLPAREN2, "(",
163
+ :tIDENTIFIER, "a",
164
+ :tSYMBOL, "b")
165
+ end
166
+
167
+ def test_yylex_label__19
168
+ setup_lexer 19
169
+
170
+ util_lex_token("{a:b",
171
+ :tLBRACE, "{",
172
+ :tLABEL, "a",
173
+ :tIDENTIFIER, "b")
174
+ end
175
+
176
+ def test_yylex_label_in_params__19
177
+ setup_lexer 19
178
+
179
+ util_lex_token("foo(a:b",
180
+ :tIDENTIFIER, "foo",
181
+ :tLPAREN2, "(",
182
+ :tLABEL, "a",
183
+ :tIDENTIFIER, "b")
184
+ end
185
+
186
+ def test_yylex_command_start__19
187
+ setup_lexer 19
188
+
189
+ %w[case elsif for in until when while
190
+ if unless and or].each do |keyword|
191
+ token = "k#{keyword.upcase}".to_sym
192
+
193
+ @lex.reset
194
+ util_lex_token("#{keyword} a:b",
195
+ token, keyword,
196
+ :tIDENTIFIER, "a",
197
+ :tSYMBOL, "b")
198
+ end
199
+ end
200
+
201
+ def test_yylex_mod_not_command_start__19
202
+ setup_lexer 19
203
+
204
+ %w[if unless while until rescue].each do |keyword|
205
+ token = "k#{keyword.upcase}_MOD".to_sym
206
+
207
+ @lex.state = :expr_end
208
+ util_lex_token("#{keyword} a:b",
209
+ token, keyword,
210
+ :tLABEL, "a",
211
+ :tIDENTIFIER, "b")
212
+ end
213
+ end
214
+
215
+ def test_yylex_back_ref
216
+ util_lex_token("[$&, $`, $', $+]",
217
+ :tLBRACK, "[",
218
+ :tBACK_REF, :"&", :tCOMMA, ",",
219
+ :tBACK_REF, :"`", :tCOMMA, ",",
220
+ :tBACK_REF, :"'", :tCOMMA, ",",
221
+ :tBACK_REF, :"+",
222
+ :tRBRACK, "]")
223
+ end
224
+
225
+ def test_yylex_backslash
226
+ util_lex_token("1 \\\n+ 2",
227
+ :tINTEGER, 1,
228
+ :tPLUS, "+",
229
+ :tINTEGER, 2)
230
+ end
231
+
232
+ def test_yylex_backslash_bad
233
+ util_bad_token("1 \\ + 2",
234
+ :tINTEGER, 1)
235
+ end
236
+
237
+ def test_yylex_backtick
238
+ util_lex_token("`ls`",
239
+ :tXSTRING_BEG, "`",
240
+ :tSTRING_CONTENT, "ls",
241
+ :tSTRING_END, "`")
242
+ end
243
+
244
+ def test_yylex_backtick_cmdarg
245
+ @lex.state = :expr_dot
246
+ util_lex_token("\n`", :tBACK_REF2, "`") # \n ensures expr_cmd
247
+
248
+ assert_equal :expr_arg, @lex.state
249
+ end
250
+
251
+ def test_yylex_backtick_dot
252
+ @lex.state = :expr_dot
253
+ util_lex_token("a.`(3)",
254
+ :tIDENTIFIER, "a",
255
+ :tDOT, ".",
256
+ :tBACK_REF2, "`",
257
+ :tLPAREN2, "(",
258
+ :tINTEGER, 3,
259
+ :tRPAREN, ")")
260
+ end
261
+
262
+ def test_yylex_backtick_method
263
+ @lex.state = :expr_fname
264
+ util_lex_token("`", :tBACK_REF2, "`")
265
+ assert_equal :expr_end, @lex.state
266
+ end
267
+
268
+ def test_yylex_bad_char
269
+ util_bad_token(" \010 ")
270
+ end
271
+
272
+ def test_yylex_bang
273
+ util_lex_token "!", :tBANG, "!"
274
+ end
275
+
276
+ def test_yylex_bang_equals
277
+ util_lex_token "!=", :tNEQ, "!="
278
+ end
279
+
280
+ def test_yylex_bang_tilde
281
+ util_lex_token "!~", :tNMATCH, "!~"
282
+ end
283
+
284
+ def test_yylex_carat
285
+ util_lex_token "^", :tCARET, "^"
286
+ end
287
+
288
+ def test_yylex_carat_equals
289
+ util_lex_token "^=", :tOP_ASGN, "^"
290
+ end
291
+
292
+ def test_yylex_colon2
293
+ util_lex_token("A::B",
294
+ :tCONSTANT, "A",
295
+ :tCOLON2, "::",
296
+ :tCONSTANT, "B")
297
+ end
298
+
299
+ def test_yylex_colon3
300
+ util_lex_token("::Array",
301
+ :tCOLON3, "::",
302
+ :tCONSTANT, "Array")
303
+ end
304
+
305
+ def test_yylex_comma
306
+ util_lex_token ",", :tCOMMA, ","
307
+ end
308
+
309
+ def test_yylex_comment
310
+ util_lex_token("1 # one\n# two\n2",
311
+ :tINTEGER, 1,
312
+ :tNL, nil,
313
+ :tINTEGER, 2)
314
+ assert_equal "# one\n# two\n", @lex.comments
315
+ end
316
+
317
+ def test_yylex_comment_begin
318
+ util_lex_token("=begin\nblah\nblah\n=end\n42",
319
+ :tINTEGER, 42)
320
+ assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
321
+ end
322
+
323
+ def test_yylex_comment_begin_bad
324
+ util_bad_token("=begin\nblah\nblah\n")
325
+ assert_equal "", @lex.comments
326
+ end
327
+
328
+ def test_yylex_comment_begin_not_comment
329
+ util_lex_token("beginfoo = 5\np x \\\n=beginfoo",
330
+ :tIDENTIFIER, "beginfoo",
331
+ :tEQL, "=",
332
+ :tINTEGER, 5,
333
+ :tNL, nil,
334
+ :tIDENTIFIER, "p",
335
+ :tIDENTIFIER, "x",
336
+ :tEQL, "=",
337
+ :tIDENTIFIER, "beginfoo")
338
+ end
339
+
340
+ def test_yylex_comment_begin_space
341
+ util_lex_token("=begin blah\nblah\n=end\n")
342
+ assert_equal "=begin blah\nblah\n=end\n", @lex.comments
343
+ end
344
+
345
+ def test_yylex_comment_end_space_and_text
346
+ util_lex_token("=begin blah\nblah\n=end blab\n")
347
+ assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
348
+ end
349
+
350
+ def test_yylex_comment_eos
351
+ util_lex_token("# comment")
352
+ end
353
+
354
+ def test_yylex_constant
355
+ util_lex_token("ArgumentError",
356
+ :tCONSTANT, "ArgumentError")
357
+ end
358
+
359
+ def test_yylex_constant_semi
360
+ util_lex_token("ArgumentError;",
361
+ :tCONSTANT, "ArgumentError",
362
+ :tSEMI, ";")
363
+ end
364
+
365
+ def test_yylex_cvar
366
+ util_lex_token "@@blah", :tCVAR, "@@blah"
367
+ end
368
+
369
+ def test_yylex_cvar_bad
370
+ assert_raises RubyParser::SyntaxError do
371
+ util_lex_token "@@1"
372
+ end
373
+ end
374
+
375
+ def test_yylex_div
376
+ util_lex_token("a / 2",
377
+ :tIDENTIFIER, "a",
378
+ :tDIVIDE, "/",
379
+ :tINTEGER, 2)
380
+ end
381
+
382
+ def test_yylex_div_equals
383
+ util_lex_token("a /= 2",
384
+ :tIDENTIFIER, "a",
385
+ :tOP_ASGN, "/",
386
+ :tINTEGER, 2)
387
+ end
388
+
389
+ def test_yylex_do
390
+ util_lex_token("x do 42 end",
391
+ :tIDENTIFIER, "x",
392
+ :kDO, "do",
393
+ :tINTEGER, 42,
394
+ :kEND, "end")
395
+ end
396
+
397
+ def test_yylex_do_block
398
+ @lex.state = :expr_endarg
399
+
400
+ util_lex_token("do 42 end",
401
+ :kDO_BLOCK, "do",
402
+ :tINTEGER, 42,
403
+ :kEND, "end")
404
+ end
405
+
406
+ def test_yylex_do_cond
407
+ @lex.cond.push true
408
+
409
+ util_lex_token("x do 42 end",
410
+ :tIDENTIFIER, "x",
411
+ :kDO_COND, "do",
412
+ :tINTEGER, 42,
413
+ :kEND, "end")
414
+ end
415
+
416
+ def test_yylex_dot
417
+ util_lex_token ".", :tDOT, "."
418
+ end
419
+
420
+ def test_yylex_dot2
421
+ util_lex_token "..", :tDOT2, ".."
422
+ end
423
+
424
+ def test_yylex_dot3
425
+ util_lex_token "...", :tDOT3, "..."
426
+ end
427
+
428
+ def test_yylex_equals
429
+ util_lex_token "=", :tEQL, "="
430
+ end
431
+
432
+ def test_yylex_equals2
433
+ util_lex_token "==", :tEQ, "=="
434
+ end
435
+
436
+ def test_yylex_equals3
437
+ util_lex_token "===", :tEQQ, "==="
438
+ end
439
+
440
+ def test_yylex_equals_tilde
441
+ util_lex_token "=~", :tMATCH, "=~"
442
+ end
443
+
444
+ def test_yylex_float
445
+ util_lex_token "1.0", :tFLOAT, 1.0
446
+ end
447
+
448
+ def test_yylex_float_bad_no_underscores
449
+ util_bad_token "1__0.0"
450
+ end
451
+
452
+ def test_yylex_float_bad_no_zero_leading
453
+ util_bad_token ".0"
454
+ end
455
+
456
+ def test_yylex_float_bad_trailing_underscore
457
+ util_bad_token "123_.0"
458
+ end
459
+
460
+ def test_yylex_float_call
461
+ util_lex_token("1.0.to_s",
462
+ :tFLOAT, 1.0,
463
+ :tDOT, ".",
464
+ :tIDENTIFIER, "to_s")
465
+ end
466
+
467
+ def test_yylex_float_dot_E
468
+ util_lex_token "1.0E10", :tFLOAT, 1.0e10
469
+ end
470
+
471
+ def test_yylex_float_dot_E_neg
472
+ util_lex_token("-1.0E10",
473
+ :tUMINUS_NUM, "-",
474
+ :tFLOAT, 1.0e10)
475
+ end
476
+
477
+ def test_yylex_float_dot_e
478
+ util_lex_token "1.0e10", :tFLOAT, 1.0e10
479
+ end
480
+
481
+ def test_yylex_float_dot_e_neg
482
+ util_lex_token("-1.0e10",
483
+ :tUMINUS_NUM, "-",
484
+ :tFLOAT, 1.0e10)
485
+ end
486
+
487
+ def test_yylex_float_e
488
+ util_lex_token "1e10", :tFLOAT, 1e10
489
+ end
490
+
491
+ def test_yylex_float_e_bad_trailing_underscore
492
+ util_bad_token "123_e10"
493
+ end
494
+
495
+ def test_yylex_float_e_minus
496
+ util_lex_token "1e-10", :tFLOAT, 1e-10
497
+ end
498
+
499
+ def test_yylex_float_e_neg
500
+ util_lex_token("-1e10",
501
+ :tUMINUS_NUM, "-",
502
+ :tFLOAT, 1e10)
503
+ end
504
+
505
+ def test_yylex_float_e_neg_minus
506
+ util_lex_token("-1e-10",
507
+ :tUMINUS_NUM, "-",
508
+ :tFLOAT, 1e-10)
509
+ end
510
+
511
+ def test_yylex_float_e_neg_plus
512
+ util_lex_token("-1e+10",
513
+ :tUMINUS_NUM, "-",
514
+ :tFLOAT, 1e10)
515
+ end
516
+
517
+ def test_yylex_float_e_plus
518
+ util_lex_token "1e+10", :tFLOAT, 1e10
519
+ end
520
+
521
+ def test_yylex_float_e_zero
522
+ util_lex_token "0e0", :tFLOAT, 0e0
523
+ end
524
+
525
+ def test_yylex_float_neg
526
+ util_lex_token("-1.0",
527
+ :tUMINUS_NUM, "-",
528
+ :tFLOAT, 1.0)
529
+ end
530
+
531
+ def test_yylex_ge
532
+ util_lex_token("a >= 2",
533
+ :tIDENTIFIER, "a",
534
+ :tGEQ, ">=",
535
+ :tINTEGER, 2)
536
+ end
537
+
538
+ def test_yylex_global
539
+ util_lex_token("$blah", :tGVAR, "$blah")
540
+ end
541
+
542
+ def test_yylex_global_backref
543
+ @lex.state = :expr_fname
544
+ util_lex_token("$`", :tGVAR, "$`")
545
+ end
546
+
547
+ def test_yylex_global_dash_nothing
548
+ util_lex_token("$- ", :tGVAR, "$-")
549
+ end
550
+
551
+ def test_yylex_global_dash_something
552
+ util_lex_token("$-x", :tGVAR, "$-x")
553
+ end
554
+
555
+ def test_yylex_global_number
556
+ @lex.state = :expr_fname
557
+ util_lex_token("$1", :tGVAR, "$1")
558
+ end
559
+
560
+ def test_yylex_global_number_big
561
+ @lex.state = :expr_fname
562
+ util_lex_token("$1234", :tGVAR, "$1234")
563
+ end
564
+
565
+ def test_yylex_global_other
566
+ util_lex_token("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
567
+ :tLBRACK, "[",
568
+ :tGVAR, "$~", :tCOMMA, ",",
569
+ :tGVAR, "$*", :tCOMMA, ",",
570
+ :tGVAR, "$$", :tCOMMA, ",",
571
+ :tGVAR, "$\?", :tCOMMA, ",",
572
+ :tGVAR, "$!", :tCOMMA, ",",
573
+ :tGVAR, "$@", :tCOMMA, ",",
574
+ :tGVAR, "$/", :tCOMMA, ",",
575
+ :tGVAR, "$\\", :tCOMMA, ",",
576
+ :tGVAR, "$;", :tCOMMA, ",",
577
+ :tGVAR, "$,", :tCOMMA, ",",
578
+ :tGVAR, "$.", :tCOMMA, ",",
579
+ :tGVAR, "$=", :tCOMMA, ",",
580
+ :tGVAR, "$:", :tCOMMA, ",",
581
+ :tGVAR, "$<", :tCOMMA, ",",
582
+ :tGVAR, "$>", :tCOMMA, ",",
583
+ :tGVAR, "$\"",
584
+ :tRBRACK, "]")
585
+ end
586
+
587
+ def test_yylex_global_underscore
588
+ util_lex_token("$_",
589
+ :tGVAR, "$_")
590
+ end
591
+
592
+ def test_yylex_global_wierd
593
+ util_lex_token("$__blah",
594
+ :tGVAR, "$__blah")
595
+ end
596
+
597
+ def test_yylex_global_zero
598
+ util_lex_token("$0", :tGVAR, "$0")
599
+ end
600
+
601
+ def test_yylex_gt
602
+ util_lex_token("a > 2",
603
+ :tIDENTIFIER, "a",
604
+ :tGT, ">",
605
+ :tINTEGER, 2)
606
+ end
607
+
608
+ def test_yylex_heredoc_backtick
609
+ util_lex_token("a = <<`EOF`\n blah blah\nEOF\n",
610
+ :tIDENTIFIER, "a",
611
+ :tEQL, "=",
612
+ :tXSTRING_BEG, "`",
613
+ :tSTRING_CONTENT, " blah blah\n",
614
+ :tSTRING_END, "EOF",
615
+ :tNL, nil)
616
+ end
617
+
618
+ def test_yylex_heredoc_double
619
+ util_lex_token("a = <<\"EOF\"\n blah blah\nEOF\n",
620
+ :tIDENTIFIER, "a",
621
+ :tEQL, "=",
622
+ :tSTRING_BEG, "\"",
623
+ :tSTRING_CONTENT, " blah blah\n",
624
+ :tSTRING_END, "EOF",
625
+ :tNL, nil)
626
+ end
627
+
628
+ def test_yylex_heredoc_double_dash
629
+ util_lex_token("a = <<-\"EOF\"\n blah blah\n EOF\n",
630
+ :tIDENTIFIER, "a",
631
+ :tEQL, "=",
632
+ :tSTRING_BEG, "\"",
633
+ :tSTRING_CONTENT, " blah blah\n",
634
+ :tSTRING_END, "EOF",
635
+ :tNL, nil)
636
+ end
637
+
638
+ def test_yylex_heredoc_double_eos
639
+ util_bad_token("a = <<\"EOF\"\nblah",
640
+ :tIDENTIFIER, "a",
641
+ :tEQL, "=",
642
+ :tSTRING_BEG, "\"")
643
+ end
644
+
645
+ def test_yylex_heredoc_double_eos_nl
646
+ util_bad_token("a = <<\"EOF\"\nblah\n",
647
+ :tIDENTIFIER, "a",
648
+ :tEQL, "=",
649
+ :tSTRING_BEG, "\"")
650
+ end
651
+
652
+ def test_yylex_heredoc_double_interp
653
+ util_lex_token("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
654
+ :tIDENTIFIER, "a",
655
+ :tEQL, "=",
656
+ :tSTRING_BEG, "\"",
657
+ :tSTRING_CONTENT, "#x a ",
658
+ :tSTRING_DVAR, nil,
659
+ :tIVAR, "@a",
660
+ :tSTRING_CONTENT, " b ",
661
+ :tSTRING_DVAR, nil,
662
+ :tGVAR, "$b",
663
+ :tSTRING_CONTENT, " c ",
664
+ :tSTRING_DBEG, '#{',
665
+ :tINTEGER, 3,
666
+ :tRCURLY, "}",
667
+ :tSTRING_CONTENT, " \n",
668
+ :tSTRING_END, "EOF",
669
+ :tNL, nil)
670
+ end
671
+
672
+ def test_yylex_heredoc_empty
673
+ util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
674
+ :tSTRING_BEG, "\"",
675
+ :tSTRING_DBEG, "\#{",
676
+ :tIDENTIFIER, "x",
677
+ :tRCURLY, "}",
678
+ :tSTRING_CONTENT, "\n",
679
+ :tSTRING_CONTENT, "blah2\n",
680
+ :tSTRING_END, "",
681
+ :tNL, nil)
682
+ end
683
+
684
+ def test_yylex_heredoc_none
685
+ util_lex_token("a = <<EOF\nblah\nblah\nEOF",
686
+ :tIDENTIFIER, "a",
687
+ :tEQL, "=",
688
+ :tSTRING_BEG, "\"",
689
+ :tSTRING_CONTENT, "blah\n",
690
+ :tSTRING_CONTENT, "blah\n",
691
+ :tSTRING_END, "EOF",
692
+ :tNL, nil)
693
+ end
694
+
695
+ def test_yylex_heredoc_none_dash
696
+ util_lex_token("a = <<-EOF\nblah\nblah\n EOF",
697
+ :tIDENTIFIER, "a",
698
+ :tEQL, "=",
699
+ :tSTRING_BEG, "\"",
700
+ :tSTRING_CONTENT, "blah\n",
701
+ :tSTRING_CONTENT, "blah\n",
702
+ :tSTRING_END, "EOF",
703
+ :tNL, nil)
704
+ end
705
+
706
+ def test_yylex_heredoc_single
707
+ util_lex_token("a = <<'EOF'\n blah blah\nEOF\n",
708
+ :tIDENTIFIER, "a",
709
+ :tEQL, "=",
710
+ :tSTRING_BEG, "'",
711
+ :tSTRING_CONTENT, " blah blah\n",
712
+ :tSTRING_END, "EOF",
713
+ :tNL, nil)
714
+ end
715
+
716
+ def test_yylex_heredoc_single_bad_eos_body
717
+ util_bad_token("a = <<'EOF'\nblah",
718
+ :tIDENTIFIER, "a",
719
+ :tEQL, "=",
720
+ :tSTRING_BEG, "'")
721
+ end
722
+
723
+ def test_yylex_heredoc_single_dash
724
+ util_lex_token("a = <<-'EOF'\n blah blah\n EOF\n",
725
+ :tIDENTIFIER, "a",
726
+ :tEQL, "=",
727
+ :tSTRING_BEG, "'",
728
+ :tSTRING_CONTENT, " blah blah\n",
729
+ :tSTRING_END, "EOF",
730
+ :tNL, nil)
731
+ end
732
+
733
+ def test_yylex_identifier
734
+ util_lex_token("identifier", :tIDENTIFIER, "identifier")
735
+ end
736
+
737
+ def test_yylex_identifier_bang
738
+ util_lex_token("identifier!", :tFID, "identifier!")
739
+ end
740
+
741
+ def test_yylex_identifier_cmp
742
+ util_lex_fname "<=>", :tCMP
743
+ end
744
+
745
+ def test_yylex_identifier_def
746
+ util_lex_fname "identifier", :tIDENTIFIER, :expr_end
747
+ end
748
+
749
+ def test_yylex_identifier_eh
750
+ util_lex_token("identifier?", :tFID, "identifier?")
751
+ end
752
+
753
+ def test_yylex_identifier_equals_arrow
754
+ util_lex_token(":blah==>",
755
+ :tSYMBOL, "blah=",
756
+ :tASSOC, "=>")
757
+ end
758
+
759
+ def test_yylex_identifier_equals3
760
+ util_lex_token(":a===b",
761
+ :tSYMBOL, "a",
762
+ :tEQQ, "===",
763
+ :tIDENTIFIER, "b")
764
+ end
765
+
766
+ def test_yylex_identifier_equals_equals_arrow
767
+ util_lex_token(":a==>b",
768
+ :tSYMBOL, "a=",
769
+ :tASSOC, "=>",
770
+ :tIDENTIFIER, "b")
771
+ end
772
+
773
+ def test_yylex_identifier_equals_caret
774
+ util_lex_fname "^", :tCARET
775
+ end
776
+
777
+ def test_yylex_identifier_equals_def
778
+ util_lex_fname "identifier=", :tIDENTIFIER, :expr_end
779
+ end
780
+
781
+ def test_yylex_identifier_equals_def2
782
+ util_lex_fname "==", :tEQ
783
+ end
784
+
785
+ def test_yylex_identifier_equals_expr
786
+ @lex.state = :expr_dot
787
+ util_lex_token("y = arg",
788
+ :tIDENTIFIER, "y",
789
+ :tEQL, "=",
790
+ :tIDENTIFIER, "arg")
791
+
792
+ assert_equal :expr_arg, @lex.state
793
+ end
794
+
795
+ def test_yylex_identifier_equals_or
796
+ util_lex_fname "|", :tPIPE
797
+ end
798
+
799
+ def test_yylex_identifier_equals_slash
800
+ util_lex_fname "/", :tDIVIDE
801
+ end
802
+
803
+ def test_yylex_identifier_equals_tilde
804
+ @lex.state = :expr_fname # can only set via parser's defs
805
+ util_lex_token("identifier=~",
806
+ :tIDENTIFIER, "identifier",
807
+ :tMATCH, "=~")
808
+ end
809
+
810
+ def test_yylex_identifier_gt
811
+ util_lex_fname ">", :tGT
812
+ end
813
+
814
+ def test_yylex_identifier_le
815
+ util_lex_fname "<=", :tLEQ
816
+ end
817
+
818
+ def test_yylex_identifier_lt
819
+ util_lex_fname "<", :tLT
820
+ end
821
+
822
+ def test_yylex_identifier_tilde
823
+ util_lex_fname "~", :tTILDE
824
+ end
825
+
826
+ def test_yylex_index
827
+ util_lex_fname "[]", :tAREF
828
+ end
829
+
830
+ def test_yylex_index_equals
831
+ util_lex_fname "[]=", :tASET
832
+ end
833
+
834
+ def test_yylex_integer
835
+ util_lex_token "42", :tINTEGER, 42
836
+ end
837
+
838
+ def test_yylex_integer_bin
839
+ util_lex_token "0b101010", :tINTEGER, 42
840
+ end
841
+
842
+ def test_yylex_integer_bin_bad_none
843
+ util_bad_token "0b "
844
+ end
845
+
846
+ def test_yylex_integer_bin_bad_underscores
847
+ util_bad_token "0b10__01"
848
+ end
849
+
850
+ def test_yylex_integer_dec
851
+ util_lex_token "42", :tINTEGER, 42
852
+ end
853
+
854
+ def test_yylex_integer_dec_bad_underscores
855
+ util_bad_token "42__24"
856
+ end
857
+
858
+ def test_yylex_integer_dec_d
859
+ util_lex_token "0d42", :tINTEGER, 42
860
+ end
861
+
862
+ def test_yylex_integer_dec_d_bad_none
863
+ util_bad_token "0d"
864
+ end
865
+
866
+ def test_yylex_integer_dec_d_bad_underscores
867
+ util_bad_token "0d42__24"
868
+ end
869
+
870
+ def test_yylex_question_eh_a__18
871
+ @lex = RubyLexer.new 18
872
+
873
+ util_lex_token "?a", :tINTEGER, 97
874
+ end
875
+
876
+ def test_yylex_question_eh_a__19
877
+ @lex = RubyLexer.new 19
878
+
879
+ util_lex_token '?a', :tSTRING, "a"
880
+ end
881
+
882
+ def test_yylex_question_eh_escape_M_escape_C__18
883
+ @lex = RubyLexer.new 18
884
+
885
+ util_lex_token '?\M-\C-a', :tINTEGER, 129
886
+ end
887
+
888
+ def test_yylex_question_eh_escape_M_escape_C__19
889
+ @lex = RubyLexer.new 19
890
+
891
+ util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
892
+ end
893
+
894
+ def test_yylex_integer_hex
895
+ util_lex_token "0x2a", :tINTEGER, 42
896
+ end
897
+
898
+ def test_yylex_integer_hex_bad_none
899
+ util_bad_token "0x "
900
+ end
901
+
902
+ def test_yylex_integer_hex_bad_underscores
903
+ util_bad_token "0xab__cd"
904
+ end
905
+
906
+ def test_yylex_integer_oct
907
+ util_lex_token "052", :tINTEGER, 42
908
+ end
909
+
910
+ def test_yylex_integer_oct_bad_range
911
+ util_bad_token "08"
912
+ end
913
+
914
+ def test_yylex_integer_oct_bad_underscores
915
+ util_bad_token "01__23"
916
+ end
917
+
918
+ def test_yylex_integer_oct_O
919
+ util_lex_token "0O52", :tINTEGER, 42
920
+ end
921
+
922
+ def test_yylex_integer_oct_O_bad_range
923
+ util_bad_token "0O8"
924
+ end
925
+
926
+ def test_yylex_integer_oct_O_bad_underscores
927
+ util_bad_token "0O1__23"
928
+ end
929
+
930
+ def test_yylex_integer_oct_O_not_bad_none
931
+ util_lex_token "0O ", :tINTEGER, 0
932
+ end
933
+
934
+ def test_yylex_integer_oct_o
935
+ util_lex_token "0o52", :tINTEGER, 42
936
+ end
937
+
938
+ def test_yylex_integer_oct_o_bad_range
939
+ util_bad_token "0o8"
940
+ end
941
+
942
+ def test_yylex_integer_oct_o_bad_underscores
943
+ util_bad_token "0o1__23"
944
+ end
945
+
946
+ def test_yylex_integer_oct_o_not_bad_none
947
+ util_lex_token "0o ", :tINTEGER, 0
948
+ end
949
+
950
+ def test_yylex_integer_trailing
951
+ util_lex_token("1.to_s",
952
+ :tINTEGER, 1,
953
+ :tDOT, '.',
954
+ :tIDENTIFIER, 'to_s')
955
+ end
956
+
957
+ def test_yylex_integer_underscore
958
+ util_lex_token "4_2", :tINTEGER, 42
959
+ end
960
+
961
+ def test_yylex_integer_underscore_bad
962
+ util_bad_token "4__2"
963
+ end
964
+
965
+ def test_yylex_integer_zero
966
+ util_lex_token "0", :tINTEGER, 0
967
+ end
968
+
969
+ def test_yylex_ivar
970
+ util_lex_token "@blah", :tIVAR, "@blah"
971
+ end
972
+
973
+ def test_yylex_ivar_bad
974
+ util_bad_token "@1"
975
+ end
976
+
977
+ def test_yylex_ivar_bad_0_length
978
+ util_bad_token "1+@\n", :tINTEGER, 1, :tPLUS, "+"
979
+ end
980
+
981
+ def test_yylex_keyword_expr
982
+ @lex.state = :expr_endarg
983
+
984
+ util_lex_token("if", :kIF_MOD, "if")
985
+
986
+ assert_equal :expr_beg, @lex.state
987
+ end
988
+
989
+ def test_yylex_lt
990
+ util_lex_token "<", :tLT, "<"
991
+ end
992
+
993
+ def test_yylex_lt2
994
+ util_lex_token("a <\< b",
995
+ :tIDENTIFIER, "a",
996
+ :tLSHFT, "<\<",
997
+ :tIDENTIFIER, "b")
998
+
999
+ end
1000
+
1001
+ def test_yylex_lt2_equals
1002
+ util_lex_token("a <\<= b",
1003
+ :tIDENTIFIER, "a",
1004
+ :tOP_ASGN, "<\<",
1005
+ :tIDENTIFIER, "b")
1006
+ end
1007
+
1008
+ def test_yylex_lt_equals
1009
+ util_lex_token "<=", :tLEQ, "<="
1010
+ end
1011
+
1012
+ def test_yylex_minus
1013
+ util_lex_token("1 - 2",
1014
+ :tINTEGER, 1,
1015
+ :tMINUS, "-",
1016
+ :tINTEGER, 2)
1017
+ end
1018
+
1019
+ def test_yylex_minus_equals
1020
+ @lex.state = :expr_end
1021
+
1022
+ util_lex_token "-=", :tOP_ASGN, "-"
1023
+ end
1024
+
1025
+ def test_yylex_minus_method
1026
+ @lex.state = :expr_fname
1027
+ util_lex_token "-", :tMINUS, "-"
1028
+ end
1029
+
1030
+ def test_yylex_minus_unary_method
1031
+ @lex.state = :expr_fname
1032
+ util_lex_token "-@", :tUMINUS, "-@"
1033
+ end
1034
+
1035
+ def test_yylex_minus_unary_number
1036
+ util_lex_token("-42",
1037
+ :tUMINUS_NUM, "-",
1038
+ :tINTEGER, 42)
1039
+ end
1040
+
1041
+ def test_yylex_nth_ref
1042
+ util_lex_token('[$1, $2, $3, $4, $5, $6, $7, $8, $9]',
1043
+ :tLBRACK, "[",
1044
+ :tNTH_REF, 1, :tCOMMA, ",",
1045
+ :tNTH_REF, 2, :tCOMMA, ",",
1046
+ :tNTH_REF, 3, :tCOMMA, ",",
1047
+ :tNTH_REF, 4, :tCOMMA, ",",
1048
+ :tNTH_REF, 5, :tCOMMA, ",",
1049
+ :tNTH_REF, 6, :tCOMMA, ",",
1050
+ :tNTH_REF, 7, :tCOMMA, ",",
1051
+ :tNTH_REF, 8, :tCOMMA, ",",
1052
+ :tNTH_REF, 9,
1053
+ :tRBRACK, "]")
1054
+ end
1055
+
1056
+ def test_yylex_open_bracket
1057
+ util_lex_token("(", :tLPAREN, "(")
1058
+ end
1059
+
1060
+ def test_yylex_open_bracket_cmdarg
1061
+ util_lex_token("m (", :tIDENTIFIER, "m",
1062
+ :tLPAREN_ARG, "(")
1063
+ end
1064
+
1065
+ def test_yylex_open_bracket_exprarg
1066
+ util_lex_token("m(", :tIDENTIFIER, "m",
1067
+ :tLPAREN2, "(")
1068
+ end
1069
+
1070
+ def test_yylex_open_curly_bracket
1071
+ util_lex_token("{",
1072
+ :tLBRACE, "{")
1073
+ end
1074
+
1075
+ def test_yylex_open_curly_bracket_arg
1076
+ util_lex_token("m { 3 }",
1077
+ :tIDENTIFIER, "m",
1078
+ :tLCURLY, "{",
1079
+ :tINTEGER, 3,
1080
+ :tRCURLY, "}")
1081
+ end
1082
+
1083
+ def test_yylex_open_curly_bracket_block
1084
+ @lex.state = :expr_endarg # seen m(3)
1085
+
1086
+ util_lex_token("{ 4 }",
1087
+ :tLBRACE_ARG, "{",
1088
+ :tINTEGER, 4,
1089
+ :tRCURLY, "}")
1090
+ end
1091
+
1092
+ def test_yylex_open_square_bracket_arg
1093
+ util_lex_token("m [ 3 ]",
1094
+ :tIDENTIFIER, "m",
1095
+ :tLBRACK, "[",
1096
+ :tINTEGER, 3,
1097
+ :tRBRACK, "]")
1098
+ end
1099
+
1100
+ def test_yylex_open_square_bracket_ary
1101
+ util_lex_token("[1, 2, 3]",
1102
+ :tLBRACK, "[",
1103
+ :tINTEGER, 1,
1104
+ :tCOMMA, ",",
1105
+ :tINTEGER, 2,
1106
+ :tCOMMA, ",",
1107
+ :tINTEGER, 3,
1108
+ :tRBRACK, "]")
1109
+ end
1110
+
1111
+ def test_yylex_open_square_bracket_meth
1112
+ util_lex_token("m[3]",
1113
+ :tIDENTIFIER, "m",
1114
+ :tLBRACK2, "[",
1115
+ :tINTEGER, 3,
1116
+ :tRBRACK, "]")
1117
+ end
1118
+
1119
+ def test_yylex_or
1120
+ util_lex_token "|", :tPIPE, "|"
1121
+ end
1122
+
1123
+ def test_yylex_or2
1124
+ util_lex_token "||", :tOROP, "||"
1125
+ end
1126
+
1127
+ def test_yylex_or2_equals
1128
+ util_lex_token "||=", :tOP_ASGN, "||"
1129
+ end
1130
+
1131
+ def test_yylex_or_equals
1132
+ util_lex_token "|=", :tOP_ASGN, "|"
1133
+ end
1134
+
1135
+ def test_yylex_percent
1136
+ util_lex_token("a % 2",
1137
+ :tIDENTIFIER, "a",
1138
+ :tPERCENT, "%",
1139
+ :tINTEGER, 2)
1140
+ end
1141
+
1142
+ def test_yylex_percent_equals
1143
+ util_lex_token("a %= 2",
1144
+ :tIDENTIFIER, "a",
1145
+ :tOP_ASGN, "%",
1146
+ :tINTEGER, 2)
1147
+ end
1148
+
1149
+ def test_yylex_plus
1150
+ util_lex_token("1 + 1", # TODO state?
1151
+ :tINTEGER, 1,
1152
+ :tPLUS, "+",
1153
+ :tINTEGER, 1)
1154
+ end
1155
+
1156
+ def test_yylex_plus_equals
1157
+ @lex.state = :expr_end
1158
+
1159
+ util_lex_token "+=", :tOP_ASGN, "+"
1160
+ end
1161
+
1162
+ def test_yylex_plus_method
1163
+ @lex.state = :expr_fname
1164
+ util_lex_token "+", :tPLUS, "+"
1165
+ end
1166
+
1167
+ def test_yylex_plus_unary_method
1168
+ @lex.state = :expr_fname
1169
+ util_lex_token "+@", :tUPLUS, "+@"
1170
+ end
1171
+
1172
+ def test_yylex_numbers
1173
+ util_lex_token "0b10", :tINTEGER, 2
1174
+ util_lex_token "0B10", :tINTEGER, 2
1175
+
1176
+ util_lex_token "0d10", :tINTEGER, 10
1177
+ util_lex_token "0D10", :tINTEGER, 10
1178
+
1179
+ util_lex_token "0x10", :tINTEGER, 16
1180
+ util_lex_token "0X10", :tINTEGER, 16
1181
+
1182
+ util_lex_token "0o10", :tINTEGER, 8
1183
+ util_lex_token "0O10", :tINTEGER, 8
1184
+ util_lex_token "0o", :tINTEGER, 0
1185
+ util_lex_token "0O", :tINTEGER, 0
1186
+
1187
+ util_lex_token "0o", :tINTEGER, 0
1188
+ util_lex_token "0O", :tINTEGER, 0
1189
+
1190
+ util_lex_token "0", :tINTEGER, 0
1191
+
1192
+ util_bad_token "0x"
1193
+ util_bad_token "0X"
1194
+ util_bad_token "0b"
1195
+ util_bad_token "0B"
1196
+ util_bad_token "0d"
1197
+ util_bad_token "0D"
1198
+
1199
+ util_bad_token "08"
1200
+ util_bad_token "09"
1201
+ util_bad_token "0o8"
1202
+ util_bad_token "0o9"
1203
+ util_bad_token "0O8"
1204
+ util_bad_token "0O9"
1205
+
1206
+ util_bad_token "1_e1"
1207
+ util_bad_token "1_.1"
1208
+ util_bad_token "1__1"
1209
+ end
1210
+
1211
+ def test_yylex_plus_unary_number
1212
+ util_lex_token("+42",
1213
+ :tINTEGER, 42)
1214
+ end
1215
+
1216
+ def test_yylex_question__18
1217
+ @lex = RubyLexer.new 18
1218
+
1219
+ util_lex_token "?*", :tINTEGER, 42
1220
+ end
1221
+
1222
+ def test_yylex_question__19
1223
+ @lex = RubyLexer.new 19
1224
+
1225
+ util_lex_token "?*", :tSTRING, "*"
1226
+ end
1227
+
1228
+ def test_yylex_question_bad_eos
1229
+ util_bad_token "?"
1230
+ end
1231
+
1232
+ def test_yylex_question_bad_ws
1233
+ util_lex_token "? ", :tEH, "?"
1234
+ util_lex_token "?\n", :tEH, "?"
1235
+ util_lex_token "?\t", :tEH, "?"
1236
+ util_lex_token "?\v", :tEH, "?"
1237
+ util_lex_token "?\r", :tEH, "?"
1238
+ util_lex_token "?\f", :tEH, "?"
1239
+ end
1240
+
1241
+ def test_yylex_question_ws_backslashed__18
1242
+ @lex = RubyLexer.new 18
1243
+
1244
+ @lex.state = :expr_beg
1245
+ util_lex_token "?\\ ", :tINTEGER, 32
1246
+ @lex.state = :expr_beg
1247
+ util_lex_token "?\\n", :tINTEGER, 10
1248
+ @lex.state = :expr_beg
1249
+ util_lex_token "?\\t", :tINTEGER, 9
1250
+ @lex.state = :expr_beg
1251
+ util_lex_token "?\\v", :tINTEGER, 11
1252
+ @lex.state = :expr_beg
1253
+ util_lex_token "?\\r", :tINTEGER, 13
1254
+ @lex.state = :expr_beg
1255
+ util_lex_token "?\\f", :tINTEGER, 12
1256
+ end
1257
+
1258
+ def test_yylex_question_ws_backslashed__19
1259
+ @lex = RubyLexer.new 19
1260
+
1261
+ @lex.state = :expr_beg
1262
+ util_lex_token "?\\ ", :tSTRING, " "
1263
+ @lex.state = :expr_beg
1264
+ util_lex_token "?\\n", :tSTRING, "\n"
1265
+ @lex.state = :expr_beg
1266
+ util_lex_token "?\\t", :tSTRING, "\t"
1267
+ @lex.state = :expr_beg
1268
+ util_lex_token "?\\v", :tSTRING, "\v"
1269
+ @lex.state = :expr_beg
1270
+ util_lex_token "?\\r", :tSTRING, "\r"
1271
+ @lex.state = :expr_beg
1272
+ util_lex_token "?\\f", :tSTRING, "\f"
1273
+ end
1274
+
1275
+ def test_yylex_rbracket
1276
+ util_lex_token "]", :tRBRACK, "]"
1277
+ end
1278
+
1279
+ def test_yylex_rcurly
1280
+ util_lex_token "}", :tRCURLY, "}"
1281
+ end
1282
+
1283
+ def test_yylex_regexp
1284
+ util_lex_token("/regexp/",
1285
+ :tREGEXP_BEG, "/",
1286
+ :tSTRING_CONTENT, "regexp",
1287
+ :tSTRING_END, "/",
1288
+ :tREGEXP_OPT, "")
1289
+ end
1290
+
1291
+ def test_yylex_regexp_ambiguous
1292
+ util_lex_token("method /regexp/",
1293
+ :tIDENTIFIER, "method",
1294
+ :tREGEXP_BEG, "/",
1295
+ :tSTRING_CONTENT, "regexp",
1296
+ :tSTRING_END, "/",
1297
+ :tREGEXP_OPT, "")
1298
+ end
1299
+
1300
+ def test_yylex_regexp_bad
1301
+ util_bad_token("/.*/xyz",
1302
+ :tREGEXP_BEG, "/",
1303
+ :tSTRING_CONTENT, ".*")
1304
+ end
1305
+
1306
+ def test_yylex_regexp_escape_C
1307
+ util_lex_token('/regex\\C-x/',
1308
+ :tREGEXP_BEG, "/",
1309
+ :tSTRING_CONTENT, "regex\\C-x",
1310
+ :tSTRING_END, "/",
1311
+ :tREGEXP_OPT, "")
1312
+ end
1313
+
1314
+ def test_yylex_regexp_escape_C_M
1315
+ util_lex_token('/regex\\C-\\M-x/',
1316
+ :tREGEXP_BEG, "/",
1317
+ :tSTRING_CONTENT, "regex\\C-\\M-x",
1318
+ :tSTRING_END, "/",
1319
+ :tREGEXP_OPT, "")
1320
+ end
1321
+
1322
+ def test_yylex_regexp_escape_C_M_craaaazy
1323
+ util_lex_token("/regex\\C-\\\n\\M-x/",
1324
+ :tREGEXP_BEG, "/",
1325
+ :tSTRING_CONTENT, "regex\\C-\\M-x",
1326
+ :tSTRING_END, "/",
1327
+ :tREGEXP_OPT, "")
1328
+ end
1329
+
1330
+ def test_yylex_regexp_escape_C_bad_dash
1331
+ util_bad_token '/regex\\Cx/', :tREGEXP_BEG, "/"
1332
+ end
1333
+
1334
+ def test_yylex_regexp_escape_C_bad_dash_eos
1335
+ util_bad_token '/regex\\C-/', :tREGEXP_BEG, "/"
1336
+ end
1337
+
1338
+ def test_yylex_regexp_escape_C_bad_dash_eos2
1339
+ util_bad_token '/regex\\C-', :tREGEXP_BEG, "/"
1340
+ end
1341
+
1342
+ def test_yylex_regexp_escape_C_bad_eos
1343
+ util_bad_token '/regex\\C/', :tREGEXP_BEG, "/"
1344
+ end
1345
+
1346
+ def test_yylex_regexp_escape_C_bad_eos2
1347
+ util_bad_token '/regex\\c', :tREGEXP_BEG, "/"
1348
+ end
1349
+
1350
+ def test_yylex_regexp_escape_M
1351
+ util_lex_token('/regex\\M-x/',
1352
+ :tREGEXP_BEG, "/",
1353
+ :tSTRING_CONTENT, "regex\\M-x",
1354
+ :tSTRING_END, "/",
1355
+ :tREGEXP_OPT, "")
1356
+ end
1357
+
1358
+ def test_yylex_regexp_escape_M_C
1359
+ util_lex_token('/regex\\M-\\C-x/',
1360
+ :tREGEXP_BEG, "/",
1361
+ :tSTRING_CONTENT, "regex\\M-\\C-x",
1362
+ :tSTRING_END, "/",
1363
+ :tREGEXP_OPT, "")
1364
+ end
1365
+
1366
+ def test_yylex_regexp_escape_M_bad_dash
1367
+ util_bad_token '/regex\\Mx/', :tREGEXP_BEG, "/"
1368
+ end
1369
+
1370
+ def test_yylex_regexp_escape_M_bad_dash_eos
1371
+ util_bad_token '/regex\\M-/', :tREGEXP_BEG, "/"
1372
+ end
1373
+
1374
+ def test_yylex_regexp_escape_M_bad_dash_eos2
1375
+ util_bad_token '/regex\\M-', :tREGEXP_BEG, "/"
1376
+ end
1377
+
1378
+ def test_yylex_regexp_escape_M_bad_eos
1379
+ util_bad_token '/regex\\M/', :tREGEXP_BEG, "/"
1380
+ end
1381
+
1382
+ def test_yylex_regexp_escape_backslash_slash
1383
+ util_lex_token('/\\//',
1384
+ :tREGEXP_BEG, "/",
1385
+ :tSTRING_CONTENT, '\\/',
1386
+ :tSTRING_END, "/",
1387
+ :tREGEXP_OPT, "")
1388
+ end
1389
+
1390
+ def test_yylex_regexp_escape_backslash_terminator
1391
+ util_lex_token('%r%blah\\%blah%',
1392
+ :tREGEXP_BEG, "%r",
1393
+ :tSTRING_CONTENT, "blah\\%blah",
1394
+ :tSTRING_END, "%",
1395
+ :tREGEXP_OPT, "")
1396
+ end
1397
+
1398
+ def test_yylex_regexp_escape_backslash_terminator_meta1
1399
+ util_lex_token('%r{blah\\}blah}',
1400
+ :tREGEXP_BEG, "%r",
1401
+ :tSTRING_CONTENT, "blah\\}blah",
1402
+ :tSTRING_END, "}",
1403
+ :tREGEXP_OPT, "")
1404
+ end
1405
+
1406
+ def test_yylex_regexp_escape_backslash_terminator_meta2
1407
+ util_lex_token('%r/blah\\/blah/',
1408
+ :tREGEXP_BEG, "%r",
1409
+ :tSTRING_CONTENT, "blah\\/blah",
1410
+ :tSTRING_END, "/",
1411
+ :tREGEXP_OPT, "")
1412
+ end
1413
+
1414
+ def test_yylex_regexp_escape_backslash_terminator_meta3
1415
+ util_lex_token('%r/blah\\%blah/',
1416
+ :tREGEXP_BEG, "%r",
1417
+ :tSTRING_CONTENT, "blah\\%blah",
1418
+ :tSTRING_END, "/",
1419
+ :tREGEXP_OPT, "")
1420
+ end
1421
+
1422
+ def test_yylex_regexp_escape_bad_eos
1423
+ util_bad_token '/regex\\', :tREGEXP_BEG, "/"
1424
+ end
1425
+
1426
+ def test_yylex_regexp_escape_bs
1427
+ util_lex_token('/regex\\\\regex/',
1428
+ :tREGEXP_BEG, "/",
1429
+ :tSTRING_CONTENT, "regex\\\\regex",
1430
+ :tSTRING_END, "/",
1431
+ :tREGEXP_OPT, "")
1432
+ end
1433
+
1434
+ def test_yylex_regexp_escape_c
1435
+ util_lex_token('/regex\\cxxx/',
1436
+ :tREGEXP_BEG, "/",
1437
+ :tSTRING_CONTENT, "regex\\cxxx",
1438
+ :tSTRING_END, "/",
1439
+ :tREGEXP_OPT, "")
1440
+ end
1441
+
1442
+ def test_yylex_regexp_escape_c_backslash
1443
+ util_lex_token('/regex\\c\\n/',
1444
+ :tREGEXP_BEG, "/",
1445
+ :tSTRING_CONTENT, "regex\\c\\n",
1446
+ :tSTRING_END, "/",
1447
+ :tREGEXP_OPT, "")
1448
+ end
1449
+
1450
+ def test_yylex_regexp_escape_chars
1451
+ util_lex_token('/re\\tge\\nxp/',
1452
+ :tREGEXP_BEG, "/",
1453
+ :tSTRING_CONTENT, "re\\tge\\nxp",
1454
+ :tSTRING_END, "/",
1455
+ :tREGEXP_OPT, "")
1456
+ end
1457
+
1458
+ def test_yylex_regexp_escape_double_backslash
1459
+ regexp = '/[\\/\\\\]$/'
1460
+ util_lex_token(regexp,
1461
+ :tREGEXP_BEG, "/",
1462
+ :tSTRING_CONTENT, regexp[1..-2],
1463
+ :tSTRING_END, "/",
1464
+ :tREGEXP_OPT, "")
1465
+ end
1466
+
1467
+ def test_yylex_regexp_escape_hex
1468
+ util_lex_token('/regex\\x61xp/',
1469
+ :tREGEXP_BEG, "/",
1470
+ :tSTRING_CONTENT, "regex\\x61xp",
1471
+ :tSTRING_END, "/",
1472
+ :tREGEXP_OPT, "")
1473
+ end
1474
+
1475
+ def test_yylex_regexp_escape_hex_bad
1476
+ util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
1477
+ end
1478
+
1479
+ def test_yylex_regexp_escape_hex_one
1480
+ util_lex_token('/^[\\xd\\xa]{2}/on',
1481
+ :tREGEXP_BEG, '/',
1482
+ :tSTRING_CONTENT, '^[\\xd\\xa]{2}',
1483
+ :tSTRING_END, "/",
1484
+ :tREGEXP_OPT, 'on')
1485
+ end
1486
+
1487
+ def test_yylex_regexp_escape_oct1
1488
+ util_lex_token('/regex\\0xp/',
1489
+ :tREGEXP_BEG, "/",
1490
+ :tSTRING_CONTENT, "regex\\0xp",
1491
+ :tSTRING_END, "/",
1492
+ :tREGEXP_OPT, "")
1493
+ end
1494
+
1495
+ def test_yylex_regexp_escape_oct2
1496
+ util_lex_token('/regex\\07xp/',
1497
+ :tREGEXP_BEG, "/",
1498
+ :tSTRING_CONTENT, "regex\\07xp",
1499
+ :tSTRING_END, "/",
1500
+ :tREGEXP_OPT, "")
1501
+ end
1502
+
1503
+ def test_yylex_regexp_escape_oct3
1504
+ util_lex_token('/regex\\10142/',
1505
+ :tREGEXP_BEG, "/",
1506
+ :tSTRING_CONTENT, "regex\\10142",
1507
+ :tSTRING_END, "/",
1508
+ :tREGEXP_OPT, "")
1509
+ end
1510
+
1511
+ def test_yylex_regexp_escape_return
1512
+ util_lex_token("/regex\\\nregex/",
1513
+ :tREGEXP_BEG, "/",
1514
+ :tSTRING_CONTENT, "regexregex",
1515
+ :tSTRING_END, "/",
1516
+ :tREGEXP_OPT, "")
1517
+ end
1518
+
1519
+ def test_yylex_regexp_nm
1520
+ util_lex_token("/.*/nm",
1521
+ :tREGEXP_BEG, "/",
1522
+ :tSTRING_CONTENT, ".*",
1523
+ :tSTRING_END, "/",
1524
+ :tREGEXP_OPT, "nm")
1525
+ end
1526
+
1527
+ def test_yylex_rparen
1528
+ util_lex_token ")", :tRPAREN, ")"
1529
+ end
1530
+
1531
+ def test_yylex_rshft
1532
+ util_lex_token("a >> 2",
1533
+ :tIDENTIFIER, "a",
1534
+ :tRSHFT, ">>",
1535
+ :tINTEGER, 2)
1536
+ end
1537
+
1538
+ def test_yylex_rshft_equals
1539
+ util_lex_token("a >>= 2",
1540
+ :tIDENTIFIER, "a",
1541
+ :tOP_ASGN, ">>",
1542
+ :tINTEGER, 2)
1543
+ end
1544
+
1545
+ def test_yylex_star
1546
+ util_lex_token("a * ",
1547
+ :tIDENTIFIER, "a",
1548
+ :tSTAR2, "*")
1549
+
1550
+ assert_equal :expr_beg, @lex.state
1551
+ end
1552
+
1553
+ def test_yylex_star2
1554
+ util_lex_token("a ** ",
1555
+ :tIDENTIFIER, "a",
1556
+ :tPOW, "**")
1557
+
1558
+ assert_equal :expr_beg, @lex.state
1559
+ end
1560
+
1561
+ def test_yylex_star2_equals
1562
+ util_lex_token("a **= ",
1563
+ :tIDENTIFIER, "a",
1564
+ :tOP_ASGN, "**")
1565
+
1566
+ assert_equal :expr_beg, @lex.state
1567
+ end
1568
+
1569
+ def test_yylex_star_arg
1570
+ @lex.state = :expr_arg
1571
+
1572
+ util_lex_token(" *a",
1573
+ :tSTAR, "*",
1574
+ :tIDENTIFIER, "a")
1575
+
1576
+ assert_equal :expr_arg, @lex.state
1577
+ end
1578
+
1579
+ def test_yylex_star_arg_beg
1580
+ @lex.state = :expr_beg
1581
+
1582
+ util_lex_token("*a",
1583
+ :tSTAR, "*",
1584
+ :tIDENTIFIER, "a")
1585
+
1586
+ assert_equal :expr_arg, @lex.state
1587
+ end
1588
+
1589
+ def test_yylex_star_arg_beg_fname
1590
+ @lex.state = :expr_fname
1591
+
1592
+ util_lex_token("*a",
1593
+ :tSTAR2, "*",
1594
+ :tIDENTIFIER, "a")
1595
+
1596
+ assert_equal :expr_arg, @lex.state
1597
+ end
1598
+
1599
+ def test_yylex_star_equals
1600
+ util_lex_token("a *= ",
1601
+ :tIDENTIFIER, "a",
1602
+ :tOP_ASGN, "*")
1603
+
1604
+ assert_equal :expr_beg, @lex.state
1605
+ end
1606
+
1607
+ def test_yylex_string_bad_eos
1608
+ util_bad_token('%',
1609
+ :tSTRING_BEG, '%')
1610
+ end
1611
+
1612
+ def test_yylex_string_bad_eos_quote
1613
+ util_bad_token('%{nest',
1614
+ :tSTRING_BEG, '%}')
1615
+ end
1616
+
1617
+ def test_yylex_string_double
1618
+ util_lex_token('"string"',
1619
+ :tSTRING, "string")
1620
+ end
1621
+
1622
+ def test_yylex_string_double_escape_C
1623
+ util_lex_token('"\\C-a"',
1624
+ :tSTRING, "\001")
1625
+ end
1626
+
1627
+ def test_yylex_string_double_escape_C_backslash
1628
+ util_lex_token('"\\C-\\\\"',
1629
+ :tSTRING, "\034")
1630
+ end
1631
+
1632
+ def test_yylex_string_double_escape_C_escape
1633
+ util_lex_token('"\\C-\\M-a"',
1634
+ :tSTRING, "\201")
1635
+ end
1636
+
1637
+ def test_yylex_string_double_escape_C_question
1638
+ util_lex_token('"\\C-?"',
1639
+ :tSTRING, "\177")
1640
+ end
1641
+
1642
+ def test_yylex_string_double_escape_M
1643
+ util_lex_token('"\\M-a"',
1644
+ :tSTRING, "\341")
1645
+ end
1646
+
1647
+ def test_yylex_string_double_escape_M_backslash
1648
+ util_lex_token('"\\M-\\\\"',
1649
+ :tSTRING, "\334")
1650
+ end
1651
+
1652
+ def test_yylex_string_double_escape_M_escape
1653
+ util_lex_token('"\\M-\\C-a"',
1654
+ :tSTRING, "\201")
1655
+ end
1656
+
1657
+ def test_yylex_string_double_escape_bs1
1658
+ util_lex_token('"a\\a\\a"',
1659
+ :tSTRING, "a\a\a")
1660
+ end
1661
+
1662
+ def test_yylex_string_double_escape_bs2
1663
+ util_lex_token('"a\\\\a"',
1664
+ :tSTRING, "a\\a")
1665
+ end
1666
+
1667
+ def test_yylex_string_double_escape_c
1668
+ util_lex_token('"\\ca"',
1669
+ :tSTRING, "\001")
1670
+ end
1671
+
1672
+ def test_yylex_string_double_escape_c_escape
1673
+ util_lex_token('"\\c\\M-a"',
1674
+ :tSTRING, "\201")
1675
+ end
1676
+
1677
+ def test_yylex_string_double_escape_c_question
1678
+ util_lex_token('"\\c?"',
1679
+ :tSTRING, "\177")
1680
+ end
1681
+
1682
+ def test_yylex_string_double_escape_chars
1683
+ util_lex_token('"s\\tri\\ng"',
1684
+ :tSTRING, "s\tri\ng")
1685
+ end
1686
+
1687
+ def test_yylex_string_double_escape_hex
1688
+ util_lex_token('"n = \\x61\\x62\\x63"',
1689
+ :tSTRING, "n = abc")
1690
+ end
1691
+
1692
+ def test_yylex_string_double_escape_octal
1693
+ util_lex_token('"n = \\101\\102\\103"',
1694
+ :tSTRING, "n = ABC")
1695
+ end
1696
+
1697
+ def test_yylex_string_double_interp
1698
+ util_lex_token("\"blah #x a \#@a b \#$b c \#{3} # \"",
1699
+ :tSTRING_BEG, "\"",
1700
+ :tSTRING_CONTENT, "blah #x a ",
1701
+ :tSTRING_DVAR, nil,
1702
+ :tIVAR, "@a",
1703
+ :tSTRING_CONTENT, " b ",
1704
+ :tSTRING_DVAR, nil,
1705
+ :tGVAR, "$b",
1706
+ :tSTRING_CONTENT, " c ",
1707
+ :tSTRING_DBEG, '#{',
1708
+ :tINTEGER, 3,
1709
+ :tRCURLY, "}",
1710
+ :tSTRING_CONTENT, " # ",
1711
+ :tSTRING_END, "\"")
1712
+ end
1713
+
1714
+ def test_yylex_string_double_nested_curlies
1715
+ util_lex_token('%{nest{one{two}one}nest}',
1716
+ :tSTRING, "nest{one{two}one}nest")
1717
+ end
1718
+
1719
+ def test_yylex_string_double_no_interp
1720
+ util_lex_token("\"# blah\"", # pound first
1721
+ :tSTRING, "# blah")
1722
+
1723
+ util_lex_token("\"blah # blah\"", # pound not first
1724
+ :tSTRING, "blah # blah")
1725
+ end
1726
+
1727
+ def test_yylex_string_escape_x_single
1728
+ util_lex_token('"\\x0"',
1729
+ :tSTRING, "\000")
1730
+ end
1731
+
1732
+ def test_yylex_string_pct_Q
1733
+ util_lex_token("%Q[s1 s2]",
1734
+ :tSTRING, "s1 s2")
1735
+ end
1736
+
1737
+ def test_yylex_string_pct_W
1738
+ util_lex_token("%W[s1 s2\ns3]", # TODO: add interpolation to these
1739
+ :tWORDS_BEG, "%W",
1740
+ :tSTRING_CONTENT, "s1",
1741
+ :tSPACE, nil,
1742
+ :tSTRING_CONTENT, "s2",
1743
+ :tSPACE, nil,
1744
+ :tSTRING_CONTENT, "s3",
1745
+ :tSPACE, nil,
1746
+ :tSTRING_END, ']')
1747
+ end
1748
+
1749
+ def test_yylex_string_pct_W_bs_nl
1750
+ util_lex_token("%W[s1 \\\ns2]", # TODO: add interpolation to these
1751
+ :tWORDS_BEG, "%W",
1752
+ :tSTRING_CONTENT, "s1",
1753
+ :tSPACE, nil,
1754
+ :tSTRING_CONTENT, "\ns2",
1755
+ :tSPACE, nil,
1756
+ :tSTRING_END, ']')
1757
+ end
1758
+
1759
+ def test_yylex_string_pct_angle
1760
+ util_lex_token("%<blah>",
1761
+ :tSTRING, "blah")
1762
+ end
1763
+
1764
+ def test_yylex_string_pct_other
1765
+ util_lex_token("%%blah%",
1766
+ :tSTRING, "blah")
1767
+ end
1768
+
1769
+ def test_yylex_string_pct_w
1770
+ util_bad_token("%w[s1 s2 ",
1771
+ :tQWORDS_BEG, "%w",
1772
+ :tSTRING_CONTENT, "s1",
1773
+ :tSPACE, nil,
1774
+ :tSTRING_CONTENT, "s2",
1775
+ :tSPACE, nil)
1776
+ end
1777
+
1778
+ def test_yylex_string_pct_w_bs_nl
1779
+ util_lex_token("%w[s1 \\\ns2]",
1780
+ :tQWORDS_BEG, "%w",
1781
+ :tSTRING_CONTENT, "s1",
1782
+ :tSPACE, nil,
1783
+ :tSTRING_CONTENT, "\ns2",
1784
+ :tSPACE, nil,
1785
+ :tSTRING_END, ']')
1786
+ end
1787
+
1788
+ def test_yylex_string_pct_w_bs_sp
1789
+ util_lex_token("%w[s\\ 1 s\\ 2]",
1790
+ :tQWORDS_BEG, "%w",
1791
+ :tSTRING_CONTENT, "s 1",
1792
+ :tSPACE, nil,
1793
+ :tSTRING_CONTENT, "s 2",
1794
+ :tSPACE, nil,
1795
+ :tSTRING_END, ']')
1796
+ end
1797
+
1798
+ def test_yylex_string_pct_w_tab
1799
+ util_lex_token("%w[abc\tdef]",
1800
+ :tQWORDS_BEG, "%w",
1801
+ :tSTRING_CONTENT, "abc",
1802
+ :tSPACE, nil,
1803
+ :tSTRING_CONTENT, "def",
1804
+ :tSPACE, nil,
1805
+ :tSTRING_END, ']')
1806
+ end
1807
+
1808
+ def test_yylex_string_single
1809
+ util_lex_token("'string'",
1810
+ :tSTRING, "string")
1811
+ end
1812
+
1813
+ def test_yylex_string_single_escape_chars
1814
+ util_lex_token("'s\\tri\\ng'",
1815
+ :tSTRING, "s\\tri\\ng")
1816
+ end
1817
+
1818
+ def test_yylex_string_single_nl
1819
+ util_lex_token("'blah\\\nblah'",
1820
+ :tSTRING, "blah\\\nblah")
1821
+ end
1822
+
1823
+ def test_yylex_symbol
1824
+ util_lex_token(":symbol",
1825
+ :tSYMBOL, "symbol")
1826
+ end
1827
+
1828
+ def test_yylex_symbol_bad_zero
1829
+ util_bad_token(":\"blah\0\"",
1830
+ :tSYMBEG, ":")
1831
+ end
1832
+
1833
+ def test_yylex_symbol_double
1834
+ util_lex_token(":\"symbol\"",
1835
+ :tSYMBOL, "symbol")
1836
+ end
1837
+
1838
+ def test_yylex_symbol_single
1839
+ util_lex_token(":'symbol'",
1840
+ :tSYMBOL, "symbol")
1841
+ end
1842
+
1843
+ def test_yylex_ternary
1844
+ util_lex_token("a ? b : c",
1845
+ :tIDENTIFIER, "a",
1846
+ :tEH, "?",
1847
+ :tIDENTIFIER, "b",
1848
+ :tCOLON, ":",
1849
+ :tIDENTIFIER, "c")
1850
+
1851
+ util_lex_token("a ?b : c",
1852
+ :tIDENTIFIER, "a",
1853
+ :tINTEGER, 98,
1854
+ :tCOLON, ":",
1855
+ :tIDENTIFIER, "c")
1856
+
1857
+ util_lex_token("a ?bb : c", # GAH! MATZ!!!
1858
+ :tIDENTIFIER, "a",
1859
+ :tEH, "?",
1860
+ :tIDENTIFIER, "bb",
1861
+ :tCOLON, ":",
1862
+ :tIDENTIFIER, "c")
1863
+
1864
+ util_lex_token("42 ?", # 42 forces expr_end
1865
+ :tINTEGER, 42,
1866
+ :tEH, "?")
1867
+ end
1868
+
1869
+ def test_yylex_tilde
1870
+ util_lex_token "~", :tTILDE, "~"
1871
+ end
1872
+
1873
+ def test_yylex_tilde_unary
1874
+ @lex.state = :expr_fname
1875
+ util_lex_token "~@", :tTILDE, "~@"
1876
+ end
1877
+
1878
+ def test_yylex_uminus
1879
+ util_lex_token("-blah",
1880
+ :tUMINUS, "-",
1881
+ :tIDENTIFIER, "blah")
1882
+ end
1883
+
1884
+ def test_yylex_underscore
1885
+ util_lex_token("_var", :tIDENTIFIER, "_var")
1886
+ end
1887
+
1888
+ def test_yylex_underscore_end
1889
+ @lex.source = "__END__\n"
1890
+ tok, = @lex.advance
1891
+ deny tok
1892
+ end
1893
+
1894
+ def test_yylex_uplus
1895
+ util_lex_token("+blah",
1896
+ :tUPLUS, "+",
1897
+ :tIDENTIFIER, "blah")
1898
+ end
1899
+
1900
+ def test_yylex_if_unless_mod
1901
+ util_lex_token("return if true unless false",
1902
+ :kRETURN, "return",
1903
+ :kIF_MOD, "if",
1904
+ :kTRUE, "true",
1905
+ :kUNLESS_MOD, "unless",
1906
+ :kFALSE, "false")
1907
+ end
1908
+
1909
+ def test_yylex_if_stmt
1910
+ util_lex_token("if true\n return end",
1911
+ :kIF, "if",
1912
+ :kTRUE, "true",
1913
+ :tNL, nil,
1914
+ :kRETURN, "return",
1915
+ :kEND, "end")
1916
+ end
1917
+
1918
+ def test_yylex_static_env
1919
+ env = RubyParserStuff::Environment.new
1920
+ env[:a] = :lvar
1921
+ @lex.static_env = env
1922
+
1923
+ util_lex_token("a [42]",
1924
+ :tIDENTIFIER, "a",
1925
+ :tLBRACK2, "[",
1926
+ :tINTEGER, 42,
1927
+ :tRBRACK, "]")
1928
+ end
1929
+
1930
+ def test_zbug_float_in_decl
1931
+ util_lex_token("def initialize(u = ",
1932
+ :kDEF, "def",
1933
+ :tIDENTIFIER, "initialize",
1934
+ :tLPAREN2, "(",
1935
+ :tIDENTIFIER, "u",
1936
+ :tEQL, "=")
1937
+
1938
+ assert_equal :expr_beg, @lex.state
1939
+
1940
+ util_lex_token("0.0, s = 0.0",
1941
+ :tFLOAT, 0.0,
1942
+ :tCOMMA, ',',
1943
+ :tIDENTIFIER, "s",
1944
+ :tEQL, "=",
1945
+ :tFLOAT, 0.0)
1946
+ end
1947
+
1948
+ def test_zbug_id_equals
1949
+ util_lex_token("a =",
1950
+ :tIDENTIFIER, "a",
1951
+ :tEQL, "=")
1952
+
1953
+ assert_equal :expr_beg, @lex.state
1954
+
1955
+ util_lex_token("0.0",
1956
+ :tFLOAT, 0.0)
1957
+ end
1958
+
1959
+ def test_zbug_no_spaces_in_decl
1960
+ util_lex_token("def initialize(u=",
1961
+ :kDEF, "def",
1962
+ :tIDENTIFIER, "initialize",
1963
+ :tLPAREN2, "(",
1964
+ :tIDENTIFIER, "u",
1965
+ :tEQL, "=")
1966
+
1967
+ assert_equal :expr_beg, @lex.state
1968
+
1969
+ util_lex_token("0.0,s=0.0",
1970
+ :tFLOAT, 0.0,
1971
+ :tCOMMA, ",",
1972
+ :tIDENTIFIER, "s",
1973
+ :tEQL, "=",
1974
+ :tFLOAT, 0.0)
1975
+ end
1976
+
1977
+ ############################################################
1978
+
1979
+ def util_bad_token s, *args
1980
+ assert_raises RubyParser::SyntaxError do
1981
+ util_lex_token s, *args
1982
+ end
1983
+ end
1984
+
1985
+ def util_escape expected, input
1986
+ @lex.reset
1987
+ @lex.source = "%Q[\\#{input}]"
1988
+
1989
+ lex_token, lex_value = @lex.advance
1990
+
1991
+ if lex_value.respond_to?(:force_encoding)
1992
+ lex_value.force_encoding('ASCII-8BIT')
1993
+ end
1994
+
1995
+ assert_equal [:tSTRING, expected],
1996
+ [lex_token, lex_value],
1997
+ @lex.source
1998
+ end
1999
+
2000
+ def util_escape_bad input
2001
+ assert_raises RubyParser::SyntaxError do
2002
+ @lex.state = :expr_beg
2003
+ util_lex_token "%Q[\\#{input}]"
2004
+ end
2005
+ end
2006
+
2007
+ def util_lex_fname name, type, end_state = :expr_end
2008
+ util_lex_token("def #{name} ", :kDEF, "def", type, name)
2009
+
2010
+ assert_equal end_state, @lex.state
2011
+ end
2012
+
2013
+ def util_lex_token input, *args
2014
+ @lex.reset(false)
2015
+ @lex.source = input
2016
+
2017
+ until args.empty? do
2018
+ token, value = args.shift(2)
2019
+
2020
+ lex_token, lex_value = @lex.advance
2021
+ assert lex_token, "no more tokens"
2022
+ assert_equal [token, value], [lex_token, lex_value], input
2023
+ end
2024
+
2025
+ lex_token, lex_value = @lex.advance
2026
+ deny lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
2027
+ end
2028
+ end