ruby_parser 3.13.1 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.autotest +18 -29
- data/History.rdoc +38 -0
- data/README.rdoc +3 -3
- data/Rakefile +10 -13
- data/bin/ruby_parse +3 -1
- data/lib/ruby20_parser.rb +3042 -2866
- data/lib/ruby20_parser.y +391 -247
- data/lib/ruby21_parser.rb +3088 -2916
- data/lib/ruby21_parser.y +399 -254
- data/lib/ruby22_parser.rb +3118 -2937
- data/lib/ruby22_parser.y +400 -255
- data/lib/ruby23_parser.rb +3119 -2940
- data/lib/ruby23_parser.y +400 -255
- data/lib/ruby24_parser.rb +3089 -2905
- data/lib/ruby24_parser.y +404 -257
- data/lib/ruby25_parser.rb +3089 -2905
- data/lib/ruby25_parser.y +404 -257
- data/lib/ruby26_parser.rb +3095 -2909
- data/lib/ruby26_parser.y +410 -261
- data/lib/ruby_lexer.rb +424 -432
- data/lib/ruby_lexer.rex.rb +1 -1
- data/lib/ruby_parser.rb +27 -27
- data/lib/ruby_parser.yy +412 -262
- data/lib/ruby_parser_extras.rb +627 -406
- data/test/test_ruby_lexer.rb +1148 -1093
- data/test/test_ruby_parser.rb +2259 -1915
- data/test/test_ruby_parser_extras.rb +39 -4
- data/tools/munge.rb +1 -1
- data/tools/ripper.rb +13 -2
- metadata +8 -8
- metadata.gz.sig +0 -0
data/test/test_ruby_lexer.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# encoding: US-ASCII
|
2
|
+
|
2
3
|
# TODO: work this out
|
3
4
|
|
4
5
|
require "minitest/autorun"
|
@@ -10,8 +11,8 @@ class TestRubyLexer < Minitest::Test
|
|
10
11
|
|
11
12
|
attr_accessor :processor, :lex, :parser_class, :lex_state
|
12
13
|
|
13
|
-
alias
|
14
|
-
alias
|
14
|
+
alias lexer lex # lets me copy/paste code from parser
|
15
|
+
alias lexer= lex=
|
15
16
|
|
16
17
|
def setup
|
17
18
|
self.lex_state = EXPR_BEG
|
@@ -21,12 +22,7 @@ class TestRubyLexer < Minitest::Test
|
|
21
22
|
def setup_lexer input, exp_sexp = nil
|
22
23
|
setup_new_parser
|
23
24
|
lex.ss = RPStringScanner.new(input)
|
24
|
-
lex.lex_state =
|
25
|
-
end
|
26
|
-
|
27
|
-
def setup_new_parser
|
28
|
-
self.processor = parser_class.new
|
29
|
-
self.lex = processor.lexer
|
25
|
+
lex.lex_state = lex_state
|
30
26
|
end
|
31
27
|
|
32
28
|
def setup_lexer_class parser_class
|
@@ -35,11 +31,16 @@ class TestRubyLexer < Minitest::Test
|
|
35
31
|
setup_lexer "blah blah"
|
36
32
|
end
|
37
33
|
|
38
|
-
def
|
34
|
+
def setup_new_parser
|
35
|
+
self.processor = parser_class.new
|
36
|
+
self.lex = processor.lexer
|
37
|
+
end
|
38
|
+
|
39
|
+
def assert_lex input, exp_sexp, *args
|
39
40
|
setup_lexer input
|
40
41
|
assert_parse input, exp_sexp if exp_sexp
|
41
42
|
|
42
|
-
|
43
|
+
yield if block_given?
|
43
44
|
|
44
45
|
args.each_slice(5) do |token, value, state, paren, brace|
|
45
46
|
assert_next_lexeme token, value, state, paren, brace
|
@@ -54,14 +55,6 @@ class TestRubyLexer < Minitest::Test
|
|
54
55
|
assert_lex(input, exp_sexp, *args, &block)
|
55
56
|
end
|
56
57
|
|
57
|
-
def refute_lex input, *args # TODO: re-sort
|
58
|
-
args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
|
59
|
-
|
60
|
-
assert_raises RubyParser::SyntaxError do
|
61
|
-
assert_lex(input, nil, *args)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
58
|
def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
|
66
59
|
assert_lex3("def #{name} ",
|
67
60
|
nil,
|
@@ -73,27 +66,26 @@ class TestRubyLexer < Minitest::Test
|
|
73
66
|
def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
|
74
67
|
adv = @lex.next_token
|
75
68
|
|
76
|
-
assert adv, "no more tokens"
|
69
|
+
assert adv, "no more tokens, expecting: %p %p %p %p %p" % [token, value, state, paren, brace]
|
77
70
|
|
78
71
|
act_token, act_value = adv
|
79
72
|
|
80
73
|
msg = message {
|
81
|
-
act = [act_token, act_value, @lex.lex_state,
|
82
|
-
@lex.paren_nest, @lex.brace_nest]
|
74
|
+
act = [act_token, act_value, @lex.lex_state, @lex.paren_nest, @lex.brace_nest]
|
83
75
|
exp = [token, value, state, paren, brace]
|
84
76
|
"#{exp.inspect} vs #{act.inspect}"
|
85
77
|
}
|
86
78
|
|
87
79
|
act_value = act_value.first if Array === act_value
|
88
80
|
|
89
|
-
assert_equal token, act_token,
|
81
|
+
assert_equal token, act_token, msg
|
90
82
|
case value
|
91
83
|
when Float then
|
92
84
|
assert_in_epsilon value, act_value, 0.001, msg
|
93
85
|
when NilClass then
|
94
86
|
assert_nil act_value, msg
|
95
87
|
else
|
96
|
-
assert_equal value, act_value,
|
88
|
+
assert_equal value, act_value, msg
|
97
89
|
end
|
98
90
|
assert_equal state, @lex.lex_state, msg if state
|
99
91
|
assert_equal paren, @lex.paren_nest, msg if paren
|
@@ -116,6 +108,20 @@ class TestRubyLexer < Minitest::Test
|
|
116
108
|
end
|
117
109
|
end
|
118
110
|
|
111
|
+
def refute_lex input, *args # TODO: re-sort
|
112
|
+
args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
|
113
|
+
|
114
|
+
assert_raises RubyParser::SyntaxError do
|
115
|
+
assert_lex(input, nil, *args)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def refute_lex5 input, *args
|
120
|
+
assert_raises RubyParser::SyntaxError do
|
121
|
+
assert_lex(input, *args)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
119
125
|
def refute_lexeme
|
120
126
|
x = y = @lex.next_token
|
121
127
|
|
@@ -140,7 +146,7 @@ class TestRubyLexer < Minitest::Test
|
|
140
146
|
yield
|
141
147
|
|
142
148
|
lexer.lex_state = EXPR_ENDARG
|
143
|
-
assert_next_lexeme :tSTRING_DEND, "}",
|
149
|
+
assert_next_lexeme :tSTRING_DEND, "}", EXPR_END, 0
|
144
150
|
|
145
151
|
lexer.lex_strterm = lex_strterm
|
146
152
|
lexer.lex_state = EXPR_BEG
|
@@ -159,9 +165,20 @@ class TestRubyLexer < Minitest::Test
|
|
159
165
|
assert_nil @lex.next_token
|
160
166
|
end
|
161
167
|
|
162
|
-
def
|
163
|
-
|
164
|
-
|
168
|
+
def test_pct_w_backslashes
|
169
|
+
["\t", "\n", "\r", "\v", "\f"].each do |char|
|
170
|
+
next if !RubyLexer::HAS_ENC and char == "\v"
|
171
|
+
|
172
|
+
assert_lex("%w[foo#{char}bar]",
|
173
|
+
s(:array, s(:str, "foo"), s(:str, "bar")),
|
174
|
+
|
175
|
+
:tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
|
176
|
+
:tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
|
177
|
+
:tSPACE, nil, EXPR_BEG, 0, 0,
|
178
|
+
:tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
|
179
|
+
:tSPACE, nil, EXPR_BEG, 0, 0,
|
180
|
+
:tSTRING_END, nil, EXPR_LIT, 0, 0)
|
181
|
+
end
|
165
182
|
end
|
166
183
|
|
167
184
|
def test_read_escape
|
@@ -217,6 +234,90 @@ class TestRubyLexer < Minitest::Test
|
|
217
234
|
assert_read_escape "\230", 'M-\cx'
|
218
235
|
end
|
219
236
|
|
237
|
+
def test_ruby21_imaginary_literal
|
238
|
+
setup_lexer_class RubyParser::V21
|
239
|
+
|
240
|
+
assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
|
241
|
+
assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
|
242
|
+
assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
|
243
|
+
assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
|
244
|
+
assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
|
245
|
+
assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
|
246
|
+
assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
|
247
|
+
assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
|
248
|
+
assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
|
249
|
+
assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
|
250
|
+
|
251
|
+
c010 = Complex(0, 10)
|
252
|
+
assert_lex3("-10i", nil,
|
253
|
+
:tUMINUS_NUM, "-", EXPR_BEG,
|
254
|
+
:tIMAGINARY, c010, EXPR_NUM)
|
255
|
+
end
|
256
|
+
|
257
|
+
def test_ruby21_imaginary_literal_with_succeeding_keyword
|
258
|
+
setup_lexer_class RubyParser::V21
|
259
|
+
|
260
|
+
# 2/4 scenarios are syntax errors on all tested versions so I
|
261
|
+
# deleted them.
|
262
|
+
|
263
|
+
assert_lex3("1if", nil,
|
264
|
+
:tINTEGER, 1, EXPR_NUM,
|
265
|
+
:kIF_MOD, "if", EXPR_PAR)
|
266
|
+
assert_lex3("1.0if", nil,
|
267
|
+
:tFLOAT, 1.0, EXPR_NUM,
|
268
|
+
:kIF_MOD, "if", EXPR_PAR)
|
269
|
+
end
|
270
|
+
|
271
|
+
def test_ruby21_rational_imaginary_literal
|
272
|
+
setup_lexer_class RubyParser::V21
|
273
|
+
|
274
|
+
assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
|
275
|
+
assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
|
276
|
+
assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
|
277
|
+
assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
|
278
|
+
assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
|
279
|
+
assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
|
280
|
+
assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
|
281
|
+
assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
|
282
|
+
assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
|
283
|
+
assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
|
284
|
+
|
285
|
+
assert_lex3("-10ri", nil,
|
286
|
+
:tUMINUS_NUM, "-", EXPR_BEG,
|
287
|
+
:tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
|
288
|
+
end
|
289
|
+
|
290
|
+
def test_ruby21_rational_literal
|
291
|
+
setup_lexer_class RubyParser::V21
|
292
|
+
|
293
|
+
assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
|
294
|
+
assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
|
295
|
+
assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
|
296
|
+
assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
|
297
|
+
assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
|
298
|
+
assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
|
299
|
+
assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
|
300
|
+
assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
|
301
|
+
assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
|
302
|
+
assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
|
303
|
+
|
304
|
+
r10 = Rational(10)
|
305
|
+
assert_lex3("-10r", nil,
|
306
|
+
:tUMINUS_NUM, "-", EXPR_BEG,
|
307
|
+
:tRATIONAL, r10, EXPR_NUM)
|
308
|
+
end
|
309
|
+
|
310
|
+
def test_unicode_ident
|
311
|
+
s = "@\u1088\u1077\u1093\u1072"
|
312
|
+
assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
|
313
|
+
end
|
314
|
+
|
315
|
+
def test_why_does_ruby_hate_me?
|
316
|
+
assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
|
317
|
+
nil,
|
318
|
+
:tSTRING, %W[ Nl% \u0000 \u0000 A \u0000 999 ].join, EXPR_END)
|
319
|
+
end
|
320
|
+
|
220
321
|
def test_yylex_ambiguous_uminus
|
221
322
|
assert_lex3("m -3",
|
222
323
|
nil,
|
@@ -248,6 +349,15 @@ class TestRubyLexer < Minitest::Test
|
|
248
349
|
assert_lex3("&&=", nil, :tOP_ASGN, "&&", EXPR_BEG)
|
249
350
|
end
|
250
351
|
|
352
|
+
def test_yylex_and_arg
|
353
|
+
self.lex_state = EXPR_ARG
|
354
|
+
|
355
|
+
assert_lex3(" &y",
|
356
|
+
nil,
|
357
|
+
:tAMPER, "&", EXPR_BEG,
|
358
|
+
:tIDENTIFIER, "y", EXPR_ARG)
|
359
|
+
end
|
360
|
+
|
251
361
|
def test_yylex_and_dot
|
252
362
|
setup_lexer_class RubyParser::V23
|
253
363
|
|
@@ -272,15 +382,6 @@ class TestRubyLexer < Minitest::Test
|
|
272
382
|
:tIDENTIFIER, "y")
|
273
383
|
end
|
274
384
|
|
275
|
-
def test_yylex_and_arg
|
276
|
-
self.lex_state = EXPR_ARG
|
277
|
-
|
278
|
-
assert_lex3(" &y",
|
279
|
-
nil,
|
280
|
-
:tAMPER, "&", EXPR_BEG,
|
281
|
-
:tIDENTIFIER, "y", EXPR_ARG)
|
282
|
-
end
|
283
|
-
|
284
385
|
def test_yylex_and_equals
|
285
386
|
assert_lex3("&=", nil, :tOP_ASGN, "&", EXPR_BEG)
|
286
387
|
end
|
@@ -303,424 +404,112 @@ class TestRubyLexer < Minitest::Test
|
|
303
404
|
assert_lex3 "=>", nil, :tASSOC, "=>", EXPR_BEG
|
304
405
|
end
|
305
406
|
|
306
|
-
def
|
307
|
-
|
407
|
+
def test_yylex_back_ref
|
408
|
+
assert_lex3("[$&, $`, $', $+]",
|
409
|
+
nil,
|
410
|
+
:tLBRACK, "[", EXPR_PAR,
|
411
|
+
:tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
412
|
+
:tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
413
|
+
:tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
414
|
+
:tBACK_REF, :+, EXPR_END,
|
415
|
+
:tRBRACK, "]", EXPR_END)
|
416
|
+
end
|
308
417
|
|
309
|
-
|
418
|
+
def test_yylex_backslash
|
419
|
+
assert_lex3("1 \\\n+ 2",
|
310
420
|
nil,
|
311
|
-
:
|
312
|
-
:
|
421
|
+
:tINTEGER, 1, EXPR_NUM,
|
422
|
+
:tPLUS, "+", EXPR_BEG,
|
423
|
+
:tINTEGER, 2, EXPR_NUM)
|
313
424
|
end
|
314
425
|
|
315
|
-
def
|
316
|
-
|
426
|
+
def test_yylex_backslash_bad
|
427
|
+
refute_lex("1 \\ + 2", :tINTEGER, 1)
|
428
|
+
end
|
317
429
|
|
318
|
-
|
430
|
+
def test_yylex_backtick
|
431
|
+
assert_lex3("`ls`",
|
319
432
|
nil,
|
320
|
-
:
|
321
|
-
:
|
322
|
-
:
|
433
|
+
:tXSTRING_BEG, "`", EXPR_BEG,
|
434
|
+
:tSTRING_CONTENT, "ls", EXPR_BEG,
|
435
|
+
:tSTRING_END, "`", EXPR_LIT)
|
323
436
|
end
|
324
437
|
|
325
|
-
def
|
326
|
-
|
327
|
-
s(:dstr,
|
328
|
-
"(",
|
329
|
-
s(:evstr, s(:call, nil, :b)),
|
330
|
-
s(:evstr, s(:call, nil, :d)),
|
331
|
-
s(:str, ")")))
|
438
|
+
def test_yylex_backtick_cmdarg
|
439
|
+
self.lex_state = EXPR_DOT
|
332
440
|
|
333
|
-
|
334
|
-
|
335
|
-
|
441
|
+
# \n ensures expr_cmd (TODO: why?)
|
442
|
+
assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
|
443
|
+
end
|
336
444
|
|
337
|
-
|
338
|
-
|
339
|
-
end
|
445
|
+
def test_yylex_backtick_dot
|
446
|
+
self.lex_state = EXPR_DOT
|
340
447
|
|
341
|
-
|
448
|
+
assert_lex3("a.`(3)",
|
449
|
+
nil,
|
450
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
451
|
+
:tDOT, ".", EXPR_DOT,
|
452
|
+
:tBACK_REF2, "`", EXPR_ARG,
|
453
|
+
:tLPAREN2, "(", EXPR_PAR,
|
454
|
+
:tINTEGER, 3, EXPR_NUM,
|
455
|
+
:tRPAREN, ")", EXPR_ENDFN)
|
456
|
+
end
|
342
457
|
|
343
|
-
|
344
|
-
|
345
|
-
end
|
458
|
+
def test_yylex_backtick_method
|
459
|
+
self.lex_state = EXPR_FNAME
|
346
460
|
|
347
|
-
|
348
|
-
|
461
|
+
assert_lex3("`",
|
462
|
+
nil,
|
463
|
+
:tBACK_REF2, "`", EXPR_END)
|
464
|
+
end
|
349
465
|
|
350
|
-
|
466
|
+
def test_yylex_bad_char
|
467
|
+
refute_lex(" \010 ")
|
351
468
|
end
|
352
469
|
|
353
|
-
def
|
354
|
-
|
355
|
-
|
470
|
+
def test_yylex_bang
|
471
|
+
assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
|
472
|
+
end
|
356
473
|
|
357
|
-
|
358
|
-
|
359
|
-
|
474
|
+
def test_yylex_bang_equals
|
475
|
+
assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
|
476
|
+
end
|
360
477
|
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
|
365
|
-
assert_next_lexeme :tREGEXP_END, "", EXPR_END, 1, 0
|
366
|
-
assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
|
367
|
-
end
|
478
|
+
def test_yylex_bang_tilde
|
479
|
+
assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
|
480
|
+
end
|
368
481
|
|
369
|
-
|
370
|
-
|
482
|
+
def test_yylex_block_bug_1
|
483
|
+
assert_lex3("a do end",
|
484
|
+
s(:iter, s(:call, nil, :a), 0),
|
371
485
|
|
372
|
-
|
486
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
487
|
+
:kDO, "do", EXPR_BEG,
|
488
|
+
:kEND, "end", EXPR_END)
|
373
489
|
end
|
374
490
|
|
375
|
-
def
|
376
|
-
|
491
|
+
def test_yylex_block_bug_2
|
492
|
+
assert_lex3("a = 1\na do\nend",
|
493
|
+
s(:block,
|
494
|
+
s(:lasgn, :a, s(:lit, 1)),
|
495
|
+
s(:iter, s(:call, nil, :a), 0)),
|
377
496
|
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
assert_lex("def !@; end",
|
387
|
-
s(:defn, :"!@", s(:args), s(:nil)),
|
388
|
-
|
389
|
-
:kDEF, "def", EXPR_FNAME, 0, 0,
|
390
|
-
:tUBANG, "!@", EXPR_ARG, 0, 0,
|
391
|
-
:tSEMI, ";", EXPR_BEG, 0, 0,
|
392
|
-
:kEND, "end", EXPR_END, 0, 0)
|
393
|
-
end
|
394
|
-
|
395
|
-
def test_yylex_not_at_ivar
|
396
|
-
assert_lex("!@ivar",
|
397
|
-
s(:call, s(:ivar, :@ivar), :"!"),
|
398
|
-
|
399
|
-
:tBANG, "!", EXPR_BEG, 0, 0,
|
400
|
-
:tIVAR, "@ivar", EXPR_END, 0, 0)
|
401
|
-
end
|
402
|
-
|
403
|
-
def test_yylex_number_times_ident_times_return_number
|
404
|
-
assert_lex("1 * b * 3",
|
405
|
-
s(:call,
|
406
|
-
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
|
407
|
-
:*, s(:lit, 3)),
|
408
|
-
|
409
|
-
:tINTEGER, 1, EXPR_NUM, 0, 0,
|
410
|
-
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
411
|
-
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
412
|
-
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
413
|
-
:tINTEGER, 3, EXPR_NUM, 0, 0)
|
414
|
-
|
415
|
-
assert_lex("1 * b *\n 3",
|
416
|
-
s(:call,
|
417
|
-
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
|
418
|
-
:*, s(:lit, 3)),
|
419
|
-
|
420
|
-
:tINTEGER, 1, EXPR_NUM, 0, 0,
|
421
|
-
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
422
|
-
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
423
|
-
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
424
|
-
:tINTEGER, 3, EXPR_NUM, 0, 0)
|
425
|
-
end
|
426
|
-
|
427
|
-
def test_yylex_paren_string_parens_interpolated_regexp
|
428
|
-
setup_lexer('%((#{(/abcd/)}))',
|
429
|
-
s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
|
430
|
-
|
431
|
-
assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
|
432
|
-
assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
|
433
|
-
|
434
|
-
assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
|
435
|
-
|
436
|
-
emulate_string_interpolation do
|
437
|
-
assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
|
438
|
-
assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
|
439
|
-
assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
|
440
|
-
assert_next_lexeme :tREGEXP_END, "", EXPR_END, 1, 0
|
441
|
-
assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
|
442
|
-
end
|
443
|
-
|
444
|
-
assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
|
445
|
-
assert_next_lexeme :tSTRING_END, ")", EXPR_END, 0, 0
|
446
|
-
|
447
|
-
refute_lexeme
|
448
|
-
end
|
449
|
-
|
450
|
-
def test_yylex_method_parens_chevron
|
451
|
-
assert_lex("a()<<1",
|
452
|
-
s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
|
453
|
-
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
454
|
-
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
455
|
-
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
456
|
-
:tLSHFT, "<<" , EXPR_BEG, 0, 0,
|
457
|
-
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
458
|
-
end
|
459
|
-
|
460
|
-
def test_yylex_lambda_args__20
|
461
|
-
setup_lexer_class RubyParser::V20
|
462
|
-
|
463
|
-
assert_lex("-> (a) { }",
|
464
|
-
s(:iter, s(:call, nil, :lambda),
|
465
|
-
s(:args, :a)),
|
466
|
-
|
467
|
-
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
468
|
-
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
469
|
-
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
470
|
-
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
471
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
472
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
473
|
-
end
|
474
|
-
|
475
|
-
def test_yylex_lambda_as_args_with_block__20
|
476
|
-
setup_lexer_class RubyParser::V20
|
477
|
-
|
478
|
-
assert_lex3("a -> do end do end",
|
479
|
-
nil,
|
480
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
481
|
-
:tLAMBDA, nil, EXPR_ENDFN,
|
482
|
-
:kDO, "do", EXPR_BEG,
|
483
|
-
:kEND, "end", EXPR_END,
|
484
|
-
:kDO, "do", EXPR_BEG,
|
485
|
-
:kEND, "end", EXPR_END)
|
486
|
-
end
|
487
|
-
|
488
|
-
def test_yylex_lambda_args_opt__20
|
489
|
-
setup_lexer_class RubyParser::V20
|
490
|
-
|
491
|
-
assert_lex("-> (a=nil) { }",
|
492
|
-
s(:iter, s(:call, nil, :lambda),
|
493
|
-
s(:args, s(:lasgn, :a, s(:nil)))),
|
494
|
-
|
495
|
-
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
496
|
-
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
497
|
-
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
498
|
-
:tEQL, "=", EXPR_BEG, 1, 0,
|
499
|
-
:kNIL, "nil", EXPR_END, 1, 0,
|
500
|
-
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
501
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
502
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
503
|
-
end
|
504
|
-
|
505
|
-
def test_yylex_lambda_hash__20
|
506
|
-
setup_lexer_class RubyParser::V20
|
507
|
-
|
508
|
-
assert_lex("-> (a={}) { }",
|
509
|
-
s(:iter, s(:call, nil, :lambda),
|
510
|
-
s(:args, s(:lasgn, :a, s(:hash)))),
|
511
|
-
|
512
|
-
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
513
|
-
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
514
|
-
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
515
|
-
:tEQL, "=", EXPR_BEG, 1, 0,
|
516
|
-
:tLBRACE, "{", EXPR_PAR, 1, 1,
|
517
|
-
:tRCURLY, "}", EXPR_ENDARG, 1, 0,
|
518
|
-
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
519
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
520
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
521
|
-
end
|
522
|
-
|
523
|
-
def test_yylex_iter_array_curly
|
524
|
-
assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
|
525
|
-
s(:iter,
|
526
|
-
s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
|
527
|
-
s(:args, :c, :d)),
|
528
|
-
|
529
|
-
:tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
|
530
|
-
:tSYMBOL, "a", EXPR_END, 0, 0,
|
531
|
-
:tCOMMA, ",", EXPR_PAR, 0, 0,
|
532
|
-
:tLBRACK, "[", EXPR_PAR, 1, 0,
|
533
|
-
:tSYMBOL, "b", EXPR_END, 1, 0,
|
534
|
-
:tRBRACK, "]", EXPR_ENDARG, 0, 0,
|
535
|
-
:tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
|
536
|
-
:tPIPE, "|", EXPR_PAR, 0, 1,
|
537
|
-
:tIDENTIFIER, "c", EXPR_ARG, 0, 1,
|
538
|
-
:tCOMMA, ",", EXPR_PAR, 0, 1,
|
539
|
-
:tIDENTIFIER, "d", EXPR_ARG, 0, 1,
|
540
|
-
:tPIPE, "|", EXPR_PAR, 0, 1,
|
541
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
542
|
-
end
|
543
|
-
|
544
|
-
def test_yylex_const_call_same_name
|
545
|
-
assert_lex("X = a { }; b { f :c }",
|
546
|
-
s(:block,
|
547
|
-
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
|
548
|
-
s(:iter,
|
549
|
-
s(:call, nil, :b),
|
550
|
-
0,
|
551
|
-
s(:call, nil, :f, s(:lit, :c)))),
|
552
|
-
|
553
|
-
:tCONSTANT, "X", EXPR_CMDARG, 0, 0,
|
554
|
-
:tEQL, "=", EXPR_BEG, 0, 0,
|
555
|
-
:tIDENTIFIER, "a", EXPR_ARG, 0, 0,
|
556
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
557
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0,
|
558
|
-
:tSEMI, ";", EXPR_BEG, 0, 0,
|
559
|
-
|
560
|
-
:tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
|
561
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
562
|
-
:tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
|
563
|
-
:tSYMBOL, "c", EXPR_END, 0, 1,
|
564
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
565
|
-
|
566
|
-
assert_lex("X = a { }; b { X :c }",
|
567
|
-
s(:block,
|
568
|
-
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
|
569
|
-
s(:iter,
|
570
|
-
s(:call, nil, :b),
|
571
|
-
0,
|
572
|
-
s(:call, nil, :X, s(:lit, :c)))),
|
573
|
-
|
574
|
-
:tCONSTANT, "X", EXPR_CMDARG, 0, 0,
|
575
|
-
:tEQL, "=", EXPR_BEG, 0, 0,
|
576
|
-
:tIDENTIFIER, "a", EXPR_ARG, 0, 0,
|
577
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
578
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0,
|
579
|
-
:tSEMI, ";", EXPR_BEG, 0, 0,
|
580
|
-
|
581
|
-
:tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
|
582
|
-
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
583
|
-
:tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
|
584
|
-
:tSYMBOL, "c", EXPR_END, 0, 1,
|
585
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
586
|
-
end
|
587
|
-
|
588
|
-
def test_yylex_lasgn_call_same_name
|
589
|
-
assert_lex("a = b.c :d => 1",
|
590
|
-
s(:lasgn, :a,
|
591
|
-
s(:call, s(:call, nil, :b), :c,
|
592
|
-
s(:hash, s(:lit, :d), s(:lit, 1)))),
|
593
|
-
|
594
|
-
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
595
|
-
:tEQL, "=", EXPR_BEG, 0, 0,
|
596
|
-
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
597
|
-
:tDOT, ".", EXPR_DOT, 0, 0,
|
598
|
-
:tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
|
599
|
-
:tSYMBOL, "d", EXPR_END, 0, 0,
|
600
|
-
:tASSOC, "=>", EXPR_BEG, 0, 0,
|
601
|
-
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
602
|
-
|
603
|
-
assert_lex("a = b.a :d => 1",
|
604
|
-
s(:lasgn, :a,
|
605
|
-
s(:call, s(:call, nil, :b), :a,
|
606
|
-
s(:hash, s(:lit, :d), s(:lit, 1)))),
|
607
|
-
|
608
|
-
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
609
|
-
:tEQL, "=", EXPR_BEG, 0, 0,
|
610
|
-
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
611
|
-
:tDOT, ".", EXPR_DOT, 0, 0,
|
612
|
-
:tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
|
613
|
-
:tSYMBOL, "d", EXPR_END, 0, 0,
|
614
|
-
:tASSOC, "=>", EXPR_BEG, 0, 0,
|
615
|
-
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
616
|
-
end
|
617
|
-
|
618
|
-
def test_yylex_back_ref
|
619
|
-
assert_lex3("[$&, $`, $', $+]",
|
620
|
-
nil,
|
621
|
-
:tLBRACK, "[", EXPR_PAR,
|
622
|
-
:tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
623
|
-
:tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
624
|
-
:tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
625
|
-
:tBACK_REF, :+, EXPR_END,
|
626
|
-
:tRBRACK, "]", EXPR_ENDARG)
|
627
|
-
end
|
628
|
-
|
629
|
-
def test_yylex_backslash
|
630
|
-
assert_lex3("1 \\\n+ 2",
|
631
|
-
nil,
|
632
|
-
:tINTEGER, 1, EXPR_NUM,
|
633
|
-
:tPLUS, "+", EXPR_BEG,
|
634
|
-
:tINTEGER, 2, EXPR_NUM)
|
635
|
-
end
|
636
|
-
|
637
|
-
def test_yylex_backslash_bad
|
638
|
-
refute_lex("1 \\ + 2", :tINTEGER, 1)
|
639
|
-
end
|
640
|
-
|
641
|
-
def test_yylex_backtick
|
642
|
-
assert_lex3("`ls`",
|
643
|
-
nil,
|
644
|
-
:tXSTRING_BEG, "`", EXPR_BEG,
|
645
|
-
:tSTRING_CONTENT, "ls", EXPR_BEG,
|
646
|
-
:tSTRING_END, "`", EXPR_END)
|
647
|
-
end
|
648
|
-
|
649
|
-
def test_yylex_backtick_cmdarg
|
650
|
-
self.lex_state = EXPR_DOT
|
651
|
-
|
652
|
-
# \n ensures expr_cmd (TODO: why?)
|
653
|
-
assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
|
654
|
-
end
|
655
|
-
|
656
|
-
def test_yylex_backtick_dot
|
657
|
-
self.lex_state = EXPR_DOT
|
658
|
-
|
659
|
-
assert_lex3("a.`(3)",
|
660
|
-
nil,
|
661
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
662
|
-
:tDOT, ".", EXPR_DOT,
|
663
|
-
:tBACK_REF2, "`", EXPR_ARG,
|
664
|
-
:tLPAREN2, "(", EXPR_PAR,
|
665
|
-
:tINTEGER, 3, EXPR_NUM,
|
666
|
-
:tRPAREN, ")", EXPR_ENDFN)
|
667
|
-
end
|
668
|
-
|
669
|
-
def test_yylex_backtick_method
|
670
|
-
self.lex_state = EXPR_FNAME
|
671
|
-
|
672
|
-
assert_lex3("`",
|
673
|
-
nil,
|
674
|
-
:tBACK_REF2, "`", EXPR_END)
|
675
|
-
end
|
676
|
-
|
677
|
-
def test_yylex_bad_char
|
678
|
-
refute_lex(" \010 ")
|
679
|
-
end
|
680
|
-
|
681
|
-
def test_yylex_bang
|
682
|
-
assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
|
683
|
-
end
|
684
|
-
|
685
|
-
def test_yylex_bang_equals
|
686
|
-
assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
|
687
|
-
end
|
688
|
-
|
689
|
-
def test_yylex_bang_tilde
|
690
|
-
assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
|
691
|
-
end
|
692
|
-
|
693
|
-
def test_yylex_block_bug_1
|
694
|
-
assert_lex3("a do end",
|
695
|
-
s(:iter, s(:call, nil, :a), 0),
|
696
|
-
|
697
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
698
|
-
:kDO, "do", EXPR_BEG,
|
699
|
-
:kEND, "end", EXPR_END)
|
700
|
-
end
|
701
|
-
|
702
|
-
def test_yylex_block_bug_2
|
703
|
-
assert_lex3("a = 1\na do\nend",
|
704
|
-
s(:block,
|
705
|
-
s(:lasgn, :a, s(:lit, 1)),
|
706
|
-
s(:iter, s(:call, nil, :a), 0)),
|
707
|
-
|
708
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
709
|
-
:tEQL, "=", EXPR_BEG,
|
710
|
-
:tINTEGER, 1, EXPR_NUM,
|
711
|
-
:tNL, nil, EXPR_BEG,
|
712
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
713
|
-
:kDO, "do", EXPR_BEG,
|
714
|
-
:kEND, "end", EXPR_END)
|
715
|
-
end
|
497
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
498
|
+
:tEQL, "=", EXPR_BEG,
|
499
|
+
:tINTEGER, 1, EXPR_NUM,
|
500
|
+
:tNL, nil, EXPR_BEG,
|
501
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
502
|
+
:kDO, "do", EXPR_BEG,
|
503
|
+
:kEND, "end", EXPR_END)
|
504
|
+
end
|
716
505
|
|
717
506
|
def test_yylex_block_bug_3
|
718
507
|
assert_lex3("a { }",
|
719
508
|
s(:iter, s(:call, nil, :a), 0),
|
720
509
|
|
721
510
|
:tIDENTIFIER, "a", EXPR_CMDARG, # verified
|
722
|
-
:tLCURLY, "{",
|
723
|
-
:tRCURLY, "}",
|
511
|
+
:tLCURLY, "{", EXPR_PAR,
|
512
|
+
:tRCURLY, "}", EXPR_END)
|
724
513
|
end
|
725
514
|
|
726
515
|
def test_yylex_carat
|
@@ -766,9 +555,9 @@ class TestRubyLexer < Minitest::Test
|
|
766
555
|
def test_yylex_comment
|
767
556
|
assert_lex3("1 # one\n# two\n2",
|
768
557
|
nil,
|
769
|
-
:tINTEGER, 1,
|
558
|
+
:tINTEGER, 1, EXPR_NUM,
|
770
559
|
:tNL, nil, EXPR_BEG,
|
771
|
-
:tINTEGER, 2,
|
560
|
+
:tINTEGER, 2, EXPR_NUM)
|
772
561
|
|
773
562
|
assert_equal "# one\n# two\n", @lex.comments
|
774
563
|
end
|
@@ -776,7 +565,7 @@ class TestRubyLexer < Minitest::Test
|
|
776
565
|
def test_yylex_comment_begin
|
777
566
|
assert_lex3("=begin\nblah\nblah\n=end\n42",
|
778
567
|
nil,
|
779
|
-
:tINTEGER, 42,
|
568
|
+
:tINTEGER, 42, EXPR_NUM)
|
780
569
|
|
781
570
|
assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
|
782
571
|
end
|
@@ -792,7 +581,7 @@ class TestRubyLexer < Minitest::Test
|
|
792
581
|
nil,
|
793
582
|
:tIDENTIFIER, "beginfoo", EXPR_CMDARG,
|
794
583
|
:tEQL, "=", EXPR_BEG,
|
795
|
-
:tINTEGER, 5,
|
584
|
+
:tINTEGER, 5, EXPR_NUM,
|
796
585
|
:tNL, nil, EXPR_BEG,
|
797
586
|
:tIDENTIFIER, "p", EXPR_CMDARG,
|
798
587
|
:tIDENTIFIER, "x", EXPR_ARG,
|
@@ -816,6 +605,50 @@ class TestRubyLexer < Minitest::Test
|
|
816
605
|
assert_lex3("# comment", nil)
|
817
606
|
end
|
818
607
|
|
608
|
+
def test_yylex_const_call_same_name
|
609
|
+
assert_lex("X = a { }; b { f :c }",
|
610
|
+
s(:block,
|
611
|
+
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
|
612
|
+
s(:iter,
|
613
|
+
s(:call, nil, :b),
|
614
|
+
0,
|
615
|
+
s(:call, nil, :f, s(:lit, :c)))),
|
616
|
+
|
617
|
+
:tCONSTANT, "X", EXPR_CMDARG, 0, 0,
|
618
|
+
:tEQL, "=", EXPR_BEG, 0, 0,
|
619
|
+
:tIDENTIFIER, "a", EXPR_ARG, 0, 0,
|
620
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
621
|
+
:tRCURLY, "}", EXPR_END, 0, 0,
|
622
|
+
:tSEMI, ";", EXPR_BEG, 0, 0,
|
623
|
+
|
624
|
+
:tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
|
625
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
626
|
+
:tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
|
627
|
+
:tSYMBOL, "c", EXPR_LIT, 0, 1,
|
628
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
629
|
+
|
630
|
+
assert_lex("X = a { }; b { X :c }",
|
631
|
+
s(:block,
|
632
|
+
s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
|
633
|
+
s(:iter,
|
634
|
+
s(:call, nil, :b),
|
635
|
+
0,
|
636
|
+
s(:call, nil, :X, s(:lit, :c)))),
|
637
|
+
|
638
|
+
:tCONSTANT, "X", EXPR_CMDARG, 0, 0,
|
639
|
+
:tEQL, "=", EXPR_BEG, 0, 0,
|
640
|
+
:tIDENTIFIER, "a", EXPR_ARG, 0, 0,
|
641
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
642
|
+
:tRCURLY, "}", EXPR_END, 0, 0,
|
643
|
+
:tSEMI, ";", EXPR_BEG, 0, 0,
|
644
|
+
|
645
|
+
:tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
|
646
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
647
|
+
:tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
|
648
|
+
:tSYMBOL, "c", EXPR_LIT, 0, 1,
|
649
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
650
|
+
end
|
651
|
+
|
819
652
|
def test_yylex_constant
|
820
653
|
assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", EXPR_CMDARG)
|
821
654
|
end
|
@@ -847,7 +680,7 @@ class TestRubyLexer < Minitest::Test
|
|
847
680
|
nil,
|
848
681
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
849
682
|
:tDIVIDE, "/", EXPR_BEG,
|
850
|
-
:tINTEGER, 2,
|
683
|
+
:tINTEGER, 2, EXPR_NUM)
|
851
684
|
end
|
852
685
|
|
853
686
|
def test_yylex_div_equals
|
@@ -855,7 +688,7 @@ class TestRubyLexer < Minitest::Test
|
|
855
688
|
nil,
|
856
689
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
857
690
|
:tOP_ASGN, "/", EXPR_BEG,
|
858
|
-
:tINTEGER, 2,
|
691
|
+
:tINTEGER, 2, EXPR_NUM)
|
859
692
|
end
|
860
693
|
|
861
694
|
def test_yylex_do
|
@@ -863,7 +696,7 @@ class TestRubyLexer < Minitest::Test
|
|
863
696
|
nil,
|
864
697
|
:tIDENTIFIER, "x", EXPR_CMDARG,
|
865
698
|
:kDO, "do", EXPR_BEG,
|
866
|
-
:tINTEGER, 42,
|
699
|
+
:tINTEGER, 42, EXPR_NUM,
|
867
700
|
:kEND, "end", EXPR_END)
|
868
701
|
end
|
869
702
|
|
@@ -876,7 +709,7 @@ class TestRubyLexer < Minitest::Test
|
|
876
709
|
:tDOT, ".", EXPR_DOT,
|
877
710
|
:tIDENTIFIER, "y", EXPR_ARG,
|
878
711
|
:kDO_BLOCK, "do", EXPR_BEG,
|
879
|
-
:tINTEGER, 42,
|
712
|
+
:tINTEGER, 42, EXPR_NUM,
|
880
713
|
:kEND, "end", EXPR_END) do
|
881
714
|
@lex.cmdarg.push true
|
882
715
|
end
|
@@ -888,24 +721,16 @@ class TestRubyLexer < Minitest::Test
|
|
888
721
|
assert_lex3("do 42 end",
|
889
722
|
nil,
|
890
723
|
:kDO_BLOCK, "do", EXPR_BEG,
|
891
|
-
:tINTEGER, 42,
|
724
|
+
:tINTEGER, 42, EXPR_NUM,
|
892
725
|
:kEND, "end", EXPR_END)
|
893
726
|
end
|
894
727
|
|
895
|
-
def test_yylex_is_your_spacebar_broken?
|
896
|
-
assert_lex3(":a!=:b",
|
897
|
-
nil,
|
898
|
-
:tSYMBOL, "a", EXPR_END,
|
899
|
-
:tNEQ, "!=", EXPR_BEG,
|
900
|
-
:tSYMBOL, "b", EXPR_END)
|
901
|
-
end
|
902
|
-
|
903
728
|
def test_yylex_do_cond
|
904
729
|
assert_lex3("x do 42 end",
|
905
730
|
nil,
|
906
731
|
:tIDENTIFIER, "x", EXPR_CMDARG,
|
907
732
|
:kDO_COND, "do", EXPR_BEG,
|
908
|
-
:tINTEGER, 42,
|
733
|
+
:tINTEGER, 42, EXPR_NUM,
|
909
734
|
:kEND, "end", EXPR_END) do
|
910
735
|
@lex.cond.push true
|
911
736
|
end
|
@@ -950,7 +775,7 @@ class TestRubyLexer < Minitest::Test
|
|
950
775
|
end
|
951
776
|
|
952
777
|
def test_yylex_float
|
953
|
-
assert_lex3("1.0", nil, :tFLOAT, 1.0,
|
778
|
+
assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
|
954
779
|
end
|
955
780
|
|
956
781
|
def test_yylex_float_bad_no_underscores
|
@@ -968,7 +793,7 @@ class TestRubyLexer < Minitest::Test
|
|
968
793
|
def test_yylex_float_call
|
969
794
|
assert_lex3("1.0.to_s",
|
970
795
|
nil,
|
971
|
-
:tFLOAT, 1.0,
|
796
|
+
:tFLOAT, 1.0, EXPR_NUM,
|
972
797
|
:tDOT, ".", EXPR_DOT,
|
973
798
|
:tIDENTIFIER, "to_s", EXPR_ARG)
|
974
799
|
end
|
@@ -983,7 +808,7 @@ class TestRubyLexer < Minitest::Test
|
|
983
808
|
assert_lex3("-1.0E10",
|
984
809
|
nil,
|
985
810
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
986
|
-
:tFLOAT, 10000000000.0,
|
811
|
+
:tFLOAT, 10000000000.0, EXPR_NUM)
|
987
812
|
end
|
988
813
|
|
989
814
|
def test_yylex_float_dot_e
|
@@ -996,29 +821,22 @@ class TestRubyLexer < Minitest::Test
|
|
996
821
|
assert_lex3("-1.0e10",
|
997
822
|
nil,
|
998
823
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
999
|
-
:tFLOAT, 10000000000.0,
|
824
|
+
:tFLOAT, 10000000000.0, EXPR_NUM)
|
1000
825
|
end
|
1001
826
|
|
1002
827
|
def test_yylex_float_e
|
1003
828
|
assert_lex3("1e10",
|
1004
829
|
nil,
|
1005
|
-
:tFLOAT, 10000000000.0,
|
830
|
+
:tFLOAT, 10000000000.0, EXPR_NUM)
|
1006
831
|
end
|
1007
832
|
|
1008
833
|
def test_yylex_float_e_bad_double_e
|
1009
834
|
assert_lex3("1e2e3",
|
1010
835
|
nil,
|
1011
|
-
:tFLOAT, 100,
|
836
|
+
:tFLOAT, 100, EXPR_NUM,
|
1012
837
|
:tIDENTIFIER, "e3", EXPR_END)
|
1013
838
|
end
|
1014
839
|
|
1015
|
-
def test_yylex_float_if_modifier
|
1016
|
-
assert_lex3("1e2if",
|
1017
|
-
nil,
|
1018
|
-
:tFLOAT, 100, EXPR_NUM,
|
1019
|
-
:kIF_MOD, "if", EXPR_PAR)
|
1020
|
-
end
|
1021
|
-
|
1022
840
|
def test_yylex_float_e_bad_trailing_underscore
|
1023
841
|
refute_lex "123_e10"
|
1024
842
|
end
|
@@ -1031,21 +849,21 @@ class TestRubyLexer < Minitest::Test
|
|
1031
849
|
assert_lex3("-1e10",
|
1032
850
|
nil,
|
1033
851
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
1034
|
-
:tFLOAT, 10000000000.0,
|
852
|
+
:tFLOAT, 10000000000.0, EXPR_NUM)
|
1035
853
|
end
|
1036
854
|
|
1037
855
|
def test_yylex_float_e_neg_minus
|
1038
856
|
assert_lex3("-1e-10",
|
1039
857
|
nil,
|
1040
858
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
1041
|
-
:tFLOAT, 1.0e-10,
|
859
|
+
:tFLOAT, 1.0e-10, EXPR_NUM)
|
1042
860
|
end
|
1043
861
|
|
1044
862
|
def test_yylex_float_e_neg_plus
|
1045
863
|
assert_lex3("-1e+10",
|
1046
864
|
nil,
|
1047
865
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
1048
|
-
:tFLOAT, 10000000000.0,
|
866
|
+
:tFLOAT, 10000000000.0, EXPR_NUM)
|
1049
867
|
end
|
1050
868
|
|
1051
869
|
def test_yylex_float_e_plus
|
@@ -1056,11 +874,18 @@ class TestRubyLexer < Minitest::Test
|
|
1056
874
|
assert_lex3("0e0", nil, :tFLOAT, 0.0, EXPR_NUM)
|
1057
875
|
end
|
1058
876
|
|
877
|
+
def test_yylex_float_if_modifier
|
878
|
+
assert_lex3("1e2if",
|
879
|
+
nil,
|
880
|
+
:tFLOAT, 100, EXPR_NUM,
|
881
|
+
:kIF_MOD, "if", EXPR_PAR)
|
882
|
+
end
|
883
|
+
|
1059
884
|
def test_yylex_float_neg
|
1060
885
|
assert_lex3("-1.0",
|
1061
886
|
nil,
|
1062
887
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
1063
|
-
:tFLOAT, 1.0,
|
888
|
+
:tFLOAT, 1.0, EXPR_NUM)
|
1064
889
|
end
|
1065
890
|
|
1066
891
|
def test_yylex_ge
|
@@ -1068,7 +893,7 @@ class TestRubyLexer < Minitest::Test
|
|
1068
893
|
nil,
|
1069
894
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1070
895
|
:tGEQ, ">=", EXPR_BEG,
|
1071
|
-
:tINTEGER, 2,
|
896
|
+
:tINTEGER, 2, EXPR_NUM)
|
1072
897
|
end
|
1073
898
|
|
1074
899
|
def test_yylex_global
|
@@ -1121,27 +946,113 @@ class TestRubyLexer < Minitest::Test
|
|
1121
946
|
:tGVAR, "$<", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
1122
947
|
:tGVAR, "$>", EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
1123
948
|
:tGVAR, "$\"", EXPR_END,
|
1124
|
-
:tRBRACK, "]",
|
949
|
+
:tRBRACK, "]", EXPR_END)
|
950
|
+
end
|
951
|
+
|
952
|
+
def test_yylex_global_underscore
|
953
|
+
assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
|
954
|
+
end
|
955
|
+
|
956
|
+
def test_yylex_global_wierd
|
957
|
+
assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
|
958
|
+
end
|
959
|
+
|
960
|
+
def test_yylex_global_zero
|
961
|
+
assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
|
962
|
+
end
|
963
|
+
|
964
|
+
def test_yylex_gt
|
965
|
+
assert_lex3("a > 2",
|
966
|
+
nil,
|
967
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
968
|
+
:tGT, ">", EXPR_BEG,
|
969
|
+
:tINTEGER, 2, EXPR_NUM)
|
970
|
+
end
|
971
|
+
|
972
|
+
def test_yylex_hash_colon
|
973
|
+
assert_lex("{a:1}",
|
974
|
+
s(:hash, s(:lit, :a), s(:lit, 1)),
|
975
|
+
|
976
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
977
|
+
:tLABEL, "a", EXPR_LAB, 0, 1,
|
978
|
+
:tINTEGER, 1, EXPR_NUM, 0, 1,
|
979
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
980
|
+
end
|
981
|
+
|
982
|
+
def test_yylex_hash_colon_double_quoted_symbol
|
983
|
+
assert_lex('{"abc": :b}',
|
984
|
+
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
985
|
+
|
986
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
987
|
+
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
988
|
+
:tSYMBOL, "b", EXPR_LIT, 0, 1,
|
989
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
990
|
+
end
|
991
|
+
|
992
|
+
def test_yylex_hash_colon_double_quoted_symbol_22
|
993
|
+
setup_lexer_class RubyParser::V22
|
994
|
+
|
995
|
+
assert_lex('{"abc": :b}',
|
996
|
+
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
997
|
+
|
998
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
999
|
+
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
1000
|
+
:tSYMBOL, "b", EXPR_LIT, 0, 1,
|
1001
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1125
1002
|
end
|
1126
1003
|
|
1127
|
-
def
|
1128
|
-
assert_lex3("
|
1004
|
+
def test_yylex_hash_colon_double_quoted_with_escapes
|
1005
|
+
assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
|
1006
|
+
nil,
|
1007
|
+
|
1008
|
+
:tLBRACE, "{", EXPR_PAR,
|
1009
|
+
:tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
|
1010
|
+
:tINTEGER, 1, EXPR_NUM,
|
1011
|
+
:tRCURLY, "}", EXPR_END)
|
1129
1012
|
end
|
1130
1013
|
|
1131
|
-
def
|
1132
|
-
|
1014
|
+
def test_yylex_hash_colon_quoted_22
|
1015
|
+
setup_lexer_class RubyParser::V22
|
1016
|
+
|
1017
|
+
assert_lex("{'a':1}",
|
1018
|
+
s(:hash, s(:lit, :a), s(:lit, 1)),
|
1019
|
+
|
1020
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
1021
|
+
:tLABEL, "a", EXPR_LAB, 0, 1,
|
1022
|
+
:tINTEGER, 1, EXPR_NUM, 0, 1,
|
1023
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1133
1024
|
end
|
1134
1025
|
|
1135
|
-
def
|
1136
|
-
|
1026
|
+
def test_yylex_hash_colon_quoted_symbol
|
1027
|
+
assert_lex("{'abc': :b}",
|
1028
|
+
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
1029
|
+
|
1030
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
1031
|
+
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
1032
|
+
:tSYMBOL, "b", EXPR_LIT, 0, 1,
|
1033
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
1137
1034
|
end
|
1138
1035
|
|
1139
|
-
def
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1036
|
+
def test_yylex_hash_colon_quoted_symbol_22
|
1037
|
+
setup_lexer_class RubyParser::V22
|
1038
|
+
|
1039
|
+
assert_lex("{'abc': :b}",
|
1040
|
+
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
1041
|
+
|
1042
|
+
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
1043
|
+
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
1044
|
+
:tSYMBOL, "b", EXPR_LIT, 0, 1,
|
1045
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1046
|
+
end
|
1047
|
+
|
1048
|
+
def test_yylex_hash_colon_quoted_with_escapes
|
1049
|
+
assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
|
1050
|
+
nil,
|
1051
|
+
|
1052
|
+
:tLBRACE, "{", EXPR_PAR,
|
1053
|
+
:tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
|
1054
|
+
:tINTEGER, 1, EXPR_NUM,
|
1055
|
+
:tRCURLY, "}", EXPR_END)
|
1145
1056
|
end
|
1146
1057
|
|
1147
1058
|
def test_yylex_heredoc_backtick
|
@@ -1151,7 +1062,7 @@ class TestRubyLexer < Minitest::Test
|
|
1151
1062
|
:tEQL, "=", EXPR_BEG,
|
1152
1063
|
:tXSTRING_BEG, "`", EXPR_BEG,
|
1153
1064
|
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1154
|
-
:tSTRING_END, "EOF",
|
1065
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1155
1066
|
:tNL, nil, EXPR_BEG)
|
1156
1067
|
end
|
1157
1068
|
|
@@ -1162,7 +1073,7 @@ class TestRubyLexer < Minitest::Test
|
|
1162
1073
|
:tEQL, "=", EXPR_BEG,
|
1163
1074
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1164
1075
|
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1165
|
-
:tSTRING_END, "EOF",
|
1076
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1166
1077
|
:tNL, nil, EXPR_BEG)
|
1167
1078
|
end
|
1168
1079
|
|
@@ -1173,51 +1084,10 @@ class TestRubyLexer < Minitest::Test
|
|
1173
1084
|
:tEQL, "=", EXPR_BEG,
|
1174
1085
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1175
1086
|
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1176
|
-
:tSTRING_END, "EOF",
|
1087
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1177
1088
|
:tNL, nil, EXPR_BEG)
|
1178
1089
|
end
|
1179
1090
|
|
1180
|
-
def test_yylex_heredoc_double_squiggly
|
1181
|
-
setup_lexer_class Ruby23Parser
|
1182
|
-
|
1183
|
-
assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
|
1184
|
-
nil,
|
1185
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1186
|
-
:tEQL, "=", EXPR_BEG,
|
1187
|
-
:tSTRING_BEG, "\"", EXPR_BEG,
|
1188
|
-
:tSTRING_CONTENT, "blah blah\n", EXPR_BEG,
|
1189
|
-
:tSTRING_END, "EOF", EXPR_END,
|
1190
|
-
:tNL, nil, EXPR_BEG)
|
1191
|
-
end
|
1192
|
-
|
1193
|
-
# mri handles tabs in a pretty specific way:
|
1194
|
-
# https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
|
1195
|
-
def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
|
1196
|
-
setup_lexer_class Ruby23Parser
|
1197
|
-
|
1198
|
-
assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
|
1199
|
-
nil,
|
1200
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1201
|
-
:tEQL, "=", EXPR_BEG,
|
1202
|
-
:tSTRING_BEG, "\"", EXPR_BEG,
|
1203
|
-
:tSTRING_CONTENT, "blah blah\n\tblah blah\n", EXPR_BEG,
|
1204
|
-
:tSTRING_END, "EOF", EXPR_END,
|
1205
|
-
:tNL, nil, EXPR_BEG)
|
1206
|
-
end
|
1207
|
-
|
1208
|
-
def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
|
1209
|
-
setup_lexer_class Ruby23Parser
|
1210
|
-
|
1211
|
-
assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
|
1212
|
-
nil,
|
1213
|
-
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1214
|
-
:tEQL, "=", EXPR_BEG,
|
1215
|
-
:tSTRING_BEG, "\"", EXPR_BEG,
|
1216
|
-
:tSTRING_CONTENT, "blah blah\n blah blah\n", EXPR_BEG,
|
1217
|
-
:tSTRING_END, "EOF", EXPR_END,
|
1218
|
-
:tNL, nil, EXPR_BEG)
|
1219
|
-
end
|
1220
|
-
|
1221
1091
|
def test_yylex_heredoc_double_eos
|
1222
1092
|
refute_lex("a = <<\"EOF\"\nblah",
|
1223
1093
|
:tIDENTIFIER, "a",
|
@@ -1247,17 +1117,30 @@ class TestRubyLexer < Minitest::Test
|
|
1247
1117
|
:tSTRING_CONTENT, "@@d ", EXPR_BEG, # HUH?
|
1248
1118
|
:tSTRING_DBEG, "\#{", EXPR_BEG,
|
1249
1119
|
:tSTRING_CONTENT, "3} \n", EXPR_BEG,
|
1250
|
-
:tSTRING_END, "EOF",
|
1120
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1251
1121
|
:tNL, nil, EXPR_BEG)
|
1252
1122
|
end
|
1253
1123
|
|
1124
|
+
def test_yylex_heredoc_double_squiggly
|
1125
|
+
setup_lexer_class Ruby23Parser
|
1126
|
+
|
1127
|
+
assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
|
1128
|
+
nil,
|
1129
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1130
|
+
:tEQL, "=", EXPR_BEG,
|
1131
|
+
:tSTRING_BEG, "\"", EXPR_BEG,
|
1132
|
+
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1133
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1134
|
+
:tNL, nil, EXPR_BEG)
|
1135
|
+
end
|
1136
|
+
|
1254
1137
|
def test_yylex_heredoc_empty
|
1255
1138
|
assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
|
1256
1139
|
nil,
|
1257
1140
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1258
1141
|
:tSTRING_DBEG, "\#{", EXPR_BEG,
|
1259
1142
|
:tSTRING_CONTENT, "x}\nblah2\n", EXPR_BEG,
|
1260
|
-
:tSTRING_END, "",
|
1143
|
+
:tSTRING_END, "", EXPR_LIT,
|
1261
1144
|
:tNL, nil, EXPR_BEG)
|
1262
1145
|
end
|
1263
1146
|
|
@@ -1268,7 +1151,7 @@ class TestRubyLexer < Minitest::Test
|
|
1268
1151
|
:tEQL, "=", EXPR_BEG,
|
1269
1152
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1270
1153
|
:tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
|
1271
|
-
:tSTRING_END, "EOF",
|
1154
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1272
1155
|
:tNL, nil, EXPR_BEG)
|
1273
1156
|
end
|
1274
1157
|
|
@@ -1286,7 +1169,7 @@ class TestRubyLexer < Minitest::Test
|
|
1286
1169
|
:tEQL, "=", EXPR_BEG,
|
1287
1170
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1288
1171
|
:tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
|
1289
|
-
:tSTRING_END, "EOF",
|
1172
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1290
1173
|
:tNL, nil, EXPR_BEG)
|
1291
1174
|
end
|
1292
1175
|
|
@@ -1298,8 +1181,8 @@ class TestRubyLexer < Minitest::Test
|
|
1298
1181
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1299
1182
|
:tEQL, "=", EXPR_BEG,
|
1300
1183
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1301
|
-
:tSTRING_CONTENT, "blah\
|
1302
|
-
:tSTRING_END, "EOF",
|
1184
|
+
:tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
|
1185
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1303
1186
|
:tNL, nil, EXPR_BEG)
|
1304
1187
|
end
|
1305
1188
|
|
@@ -1310,7 +1193,7 @@ class TestRubyLexer < Minitest::Test
|
|
1310
1193
|
:tEQL, "=", EXPR_BEG,
|
1311
1194
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1312
1195
|
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1313
|
-
:tSTRING_END, "EOF",
|
1196
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1314
1197
|
:tNL, nil, EXPR_BEG)
|
1315
1198
|
end
|
1316
1199
|
|
@@ -1349,7 +1232,7 @@ class TestRubyLexer < Minitest::Test
|
|
1349
1232
|
:tEQL, "=", EXPR_BEG,
|
1350
1233
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1351
1234
|
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1352
|
-
:tSTRING_END, "EOF",
|
1235
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1353
1236
|
:tNL, nil, EXPR_BEG)
|
1354
1237
|
end
|
1355
1238
|
|
@@ -1361,8 +1244,8 @@ class TestRubyLexer < Minitest::Test
|
|
1361
1244
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1362
1245
|
:tEQL, "=", EXPR_BEG,
|
1363
1246
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
1364
|
-
:tSTRING_CONTENT, "blah blah\n", EXPR_BEG,
|
1365
|
-
:tSTRING_END, "EOF",
|
1247
|
+
:tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
|
1248
|
+
:tSTRING_END, "EOF", EXPR_LIT,
|
1366
1249
|
:tNL, nil, EXPR_BEG)
|
1367
1250
|
end
|
1368
1251
|
|
@@ -1392,41 +1275,41 @@ class TestRubyLexer < Minitest::Test
|
|
1392
1275
|
assert_lex3("identifier?", nil, :tFID, "identifier?", EXPR_CMDARG)
|
1393
1276
|
end
|
1394
1277
|
|
1395
|
-
def test_yylex_identifier_equals_arrow
|
1396
|
-
assert_lex3(":blah==>",
|
1397
|
-
nil,
|
1398
|
-
:tSYMBOL, "blah=", EXPR_END,
|
1399
|
-
:tASSOC, "=>", EXPR_BEG)
|
1400
|
-
end
|
1401
|
-
|
1402
1278
|
def test_yylex_identifier_equals3
|
1403
1279
|
assert_lex3(":a===b",
|
1404
1280
|
nil,
|
1405
|
-
:tSYMBOL, "a",
|
1281
|
+
:tSYMBOL, "a", EXPR_LIT,
|
1406
1282
|
:tEQQ, "===", EXPR_BEG,
|
1407
1283
|
:tIDENTIFIER, "b", EXPR_ARG)
|
1408
1284
|
end
|
1409
1285
|
|
1410
|
-
def
|
1411
|
-
assert_lex3(":
|
1286
|
+
def test_yylex_identifier_equals_arrow
|
1287
|
+
assert_lex3(":blah==>",
|
1412
1288
|
nil,
|
1413
|
-
:tSYMBOL, "
|
1414
|
-
:tASSOC,
|
1415
|
-
:tIDENTIFIER, "b", EXPR_ARG)
|
1289
|
+
:tSYMBOL, "blah=", EXPR_LIT,
|
1290
|
+
:tASSOC, "=>", EXPR_BEG)
|
1416
1291
|
end
|
1417
1292
|
|
1418
1293
|
def test_yylex_identifier_equals_caret
|
1419
1294
|
assert_lex_fname "^", :tCARET
|
1420
1295
|
end
|
1421
1296
|
|
1297
|
+
def test_yylex_identifier_equals_def2
|
1298
|
+
assert_lex_fname "==", :tEQ
|
1299
|
+
end
|
1300
|
+
|
1422
1301
|
def test_yylex_identifier_equals_def__20
|
1423
1302
|
setup_lexer_class RubyParser::V20
|
1424
1303
|
|
1425
1304
|
assert_lex_fname "identifier=", :tIDENTIFIER, EXPR_ENDFN
|
1426
1305
|
end
|
1427
1306
|
|
1428
|
-
def
|
1429
|
-
|
1307
|
+
def test_yylex_identifier_equals_equals_arrow
|
1308
|
+
assert_lex3(":a==>b",
|
1309
|
+
nil,
|
1310
|
+
:tSYMBOL, "a=", EXPR_LIT,
|
1311
|
+
:tASSOC, "=>", EXPR_BEG,
|
1312
|
+
:tIDENTIFIER, "b", EXPR_ARG)
|
1430
1313
|
end
|
1431
1314
|
|
1432
1315
|
def test_yylex_identifier_equals_expr
|
@@ -1515,25 +1398,6 @@ class TestRubyLexer < Minitest::Test
|
|
1515
1398
|
refute_lex "0d42__24"
|
1516
1399
|
end
|
1517
1400
|
|
1518
|
-
def test_yylex_integer_if_modifier
|
1519
|
-
assert_lex3("123if",
|
1520
|
-
nil,
|
1521
|
-
:tINTEGER, 123, EXPR_NUM,
|
1522
|
-
:kIF_MOD, "if", EXPR_PAR)
|
1523
|
-
end
|
1524
|
-
|
1525
|
-
def test_yylex_question_eh_a__20
|
1526
|
-
setup_lexer_class RubyParser::V20
|
1527
|
-
|
1528
|
-
assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
|
1529
|
-
end
|
1530
|
-
|
1531
|
-
def test_yylex_question_eh_escape_M_escape_C__20
|
1532
|
-
setup_lexer_class RubyParser::V20
|
1533
|
-
|
1534
|
-
assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
|
1535
|
-
end
|
1536
|
-
|
1537
1401
|
def test_yylex_integer_hex
|
1538
1402
|
assert_lex3 "0x2a", nil, :tINTEGER, 42, EXPR_NUM
|
1539
1403
|
end
|
@@ -1546,20 +1410,15 @@ class TestRubyLexer < Minitest::Test
|
|
1546
1410
|
refute_lex "0xab__cd"
|
1547
1411
|
end
|
1548
1412
|
|
1549
|
-
def
|
1550
|
-
assert_lex3("
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
refute_lex "08"
|
1555
|
-
end
|
1556
|
-
|
1557
|
-
def test_yylex_integer_oct_bad_range2
|
1558
|
-
refute_lex "08"
|
1413
|
+
def test_yylex_integer_if_modifier
|
1414
|
+
assert_lex3("123if",
|
1415
|
+
nil,
|
1416
|
+
:tINTEGER, 123, EXPR_NUM,
|
1417
|
+
:kIF_MOD, "if", EXPR_PAR)
|
1559
1418
|
end
|
1560
1419
|
|
1561
|
-
def
|
1562
|
-
|
1420
|
+
def test_yylex_integer_oct
|
1421
|
+
assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
|
1563
1422
|
end
|
1564
1423
|
|
1565
1424
|
def test_yylex_integer_oct_O
|
@@ -1578,6 +1437,18 @@ class TestRubyLexer < Minitest::Test
|
|
1578
1437
|
assert_lex3 "0O ", nil, :tINTEGER, 0, EXPR_NUM
|
1579
1438
|
end
|
1580
1439
|
|
1440
|
+
def test_yylex_integer_oct_bad_range
|
1441
|
+
refute_lex "08"
|
1442
|
+
end
|
1443
|
+
|
1444
|
+
def test_yylex_integer_oct_bad_range2
|
1445
|
+
refute_lex "08"
|
1446
|
+
end
|
1447
|
+
|
1448
|
+
def test_yylex_integer_oct_bad_underscores
|
1449
|
+
refute_lex "01__23"
|
1450
|
+
end
|
1451
|
+
|
1581
1452
|
def test_yylex_integer_oct_o
|
1582
1453
|
assert_lex3 "0o52", nil, :tINTEGER, 42, EXPR_NUM
|
1583
1454
|
end
|
@@ -1590,46 +1461,247 @@ class TestRubyLexer < Minitest::Test
|
|
1590
1461
|
refute_lex "0o1__23"
|
1591
1462
|
end
|
1592
1463
|
|
1593
|
-
def test_yylex_integer_oct_o_not_bad_none
|
1594
|
-
assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
|
1464
|
+
def test_yylex_integer_oct_o_not_bad_none
|
1465
|
+
assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
|
1466
|
+
end
|
1467
|
+
|
1468
|
+
def test_yylex_integer_trailing
|
1469
|
+
assert_lex3("1.to_s",
|
1470
|
+
nil,
|
1471
|
+
:tINTEGER, 1, EXPR_NUM,
|
1472
|
+
:tDOT, ".", EXPR_DOT,
|
1473
|
+
:tIDENTIFIER, "to_s", EXPR_ARG)
|
1474
|
+
end
|
1475
|
+
|
1476
|
+
def test_yylex_integer_underscore
|
1477
|
+
assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
|
1478
|
+
end
|
1479
|
+
|
1480
|
+
def test_yylex_integer_underscore_bad
|
1481
|
+
refute_lex "4__2"
|
1482
|
+
end
|
1483
|
+
|
1484
|
+
def test_yylex_integer_zero
|
1485
|
+
assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
|
1486
|
+
end
|
1487
|
+
|
1488
|
+
def test_yylex_is_your_spacebar_broken?
|
1489
|
+
assert_lex3(":a!=:b",
|
1490
|
+
nil,
|
1491
|
+
:tSYMBOL, "a", EXPR_LIT,
|
1492
|
+
:tNEQ, "!=", EXPR_BEG,
|
1493
|
+
:tSYMBOL, "b", EXPR_LIT)
|
1494
|
+
end
|
1495
|
+
|
1496
|
+
def test_yylex_iter_array_curly
|
1497
|
+
# this will lex, but doesn't parse... don't freak out.
|
1498
|
+
assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
|
1499
|
+
nil,
|
1500
|
+
|
1501
|
+
:tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
|
1502
|
+
:tSYMBOL, "a", EXPR_LIT, 0, 0,
|
1503
|
+
:tCOMMA, ",", EXPR_PAR, 0, 0,
|
1504
|
+
:tLBRACK, "[", EXPR_PAR, 1, 0,
|
1505
|
+
:tSYMBOL, "b", EXPR_LIT, 1, 0,
|
1506
|
+
:tRBRACK, "]", EXPR_END, 0, 0,
|
1507
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1508
|
+
:tPIPE, "|", EXPR_PAR, 0, 1,
|
1509
|
+
:tIDENTIFIER, "c", EXPR_ARG, 0, 1,
|
1510
|
+
:tCOMMA, ",", EXPR_PAR, 0, 1,
|
1511
|
+
:tIDENTIFIER, "d", EXPR_ARG, 0, 1,
|
1512
|
+
:tPIPE, "|", EXPR_PAR, 0, 1,
|
1513
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
1514
|
+
end
|
1515
|
+
|
1516
|
+
def test_yylex_iter_array_curly__24
|
1517
|
+
setup_lexer_class RubyParser::V24
|
1518
|
+
|
1519
|
+
assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
|
1520
|
+
s(:iter,
|
1521
|
+
s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
|
1522
|
+
s(:args, :c, :d)),
|
1523
|
+
|
1524
|
+
:tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
|
1525
|
+
:tSYMBOL, "a", EXPR_LIT, 0, 0,
|
1526
|
+
:tCOMMA, ",", EXPR_PAR, 0, 0,
|
1527
|
+
:tLBRACK, "[", EXPR_PAR, 1, 0,
|
1528
|
+
:tSYMBOL, "b", EXPR_LIT, 1, 0,
|
1529
|
+
:tRBRACK, "]", EXPR_ENDARG, 0, 0,
|
1530
|
+
:tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
|
1531
|
+
:tPIPE, "|", EXPR_PAR, 0, 1,
|
1532
|
+
:tIDENTIFIER, "c", EXPR_ARG, 0, 1,
|
1533
|
+
:tCOMMA, ",", EXPR_PAR, 0, 1,
|
1534
|
+
:tIDENTIFIER, "d", EXPR_ARG, 0, 1,
|
1535
|
+
:tPIPE, "|", EXPR_PAR, 0, 1,
|
1536
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1537
|
+
end
|
1538
|
+
|
1539
|
+
def test_yylex_ivar
|
1540
|
+
assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
|
1541
|
+
end
|
1542
|
+
|
1543
|
+
def test_yylex_ivar_bad
|
1544
|
+
refute_lex "@1"
|
1545
|
+
end
|
1546
|
+
|
1547
|
+
def test_yylex_ivar_bad_0_length
|
1548
|
+
refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
|
1549
|
+
end
|
1550
|
+
|
1551
|
+
def test_yylex_keyword_expr
|
1552
|
+
self.lex_state = EXPR_ENDARG
|
1553
|
+
|
1554
|
+
assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
|
1555
|
+
end
|
1556
|
+
|
1557
|
+
def test_yylex_label
|
1558
|
+
assert_lex3("{a:",
|
1559
|
+
nil,
|
1560
|
+
:tLBRACE, "{", EXPR_PAR,
|
1561
|
+
:tLABEL, "a", EXPR_LAB)
|
1562
|
+
end
|
1563
|
+
|
1564
|
+
def test_yylex_label_in_params
|
1565
|
+
assert_lex3("foo(a:",
|
1566
|
+
nil,
|
1567
|
+
:tIDENTIFIER, "foo", EXPR_CMDARG,
|
1568
|
+
:tLPAREN2, "(", EXPR_PAR,
|
1569
|
+
:tLABEL, "a", EXPR_LAB)
|
1570
|
+
end
|
1571
|
+
|
1572
|
+
def test_yylex_lambda_args
|
1573
|
+
assert_lex("-> (a) { }",
|
1574
|
+
s(:iter, s(:lambda),
|
1575
|
+
s(:args, :a)),
|
1576
|
+
|
1577
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1578
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1579
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1580
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1581
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1582
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
1583
|
+
end
|
1584
|
+
|
1585
|
+
def test_yylex_lambda_args__24
|
1586
|
+
setup_lexer_class RubyParser::V24
|
1587
|
+
|
1588
|
+
assert_lex("-> (a) { }",
|
1589
|
+
s(:iter, s(:lambda),
|
1590
|
+
s(:args, :a)),
|
1591
|
+
|
1592
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1593
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1594
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1595
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1596
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1597
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1598
|
+
end
|
1599
|
+
|
1600
|
+
def test_yylex_lambda_args_opt
|
1601
|
+
assert_lex("-> (a=nil) { }",
|
1602
|
+
s(:iter, s(:lambda),
|
1603
|
+
s(:args, s(:lasgn, :a, s(:nil)))),
|
1604
|
+
|
1605
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1606
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1607
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1608
|
+
:tEQL, "=", EXPR_BEG, 1, 0,
|
1609
|
+
:kNIL, "nil", EXPR_END, 1, 0,
|
1610
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1611
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1612
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
1613
|
+
end
|
1614
|
+
|
1615
|
+
def test_yylex_lambda_args_opt__24
|
1616
|
+
setup_lexer_class RubyParser::V24
|
1617
|
+
|
1618
|
+
assert_lex("-> (a=nil) { }",
|
1619
|
+
s(:iter, s(:lambda),
|
1620
|
+
s(:args, s(:lasgn, :a, s(:nil)))),
|
1621
|
+
|
1622
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1623
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1624
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1625
|
+
:tEQL, "=", EXPR_BEG, 1, 0,
|
1626
|
+
:kNIL, "nil", EXPR_END, 1, 0,
|
1627
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1628
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1629
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1595
1630
|
end
|
1596
1631
|
|
1597
|
-
def
|
1598
|
-
assert_lex3("
|
1632
|
+
def test_yylex_lambda_as_args_with_block
|
1633
|
+
assert_lex3("a -> do end do end",
|
1599
1634
|
nil,
|
1600
|
-
:
|
1601
|
-
:
|
1602
|
-
:
|
1635
|
+
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1636
|
+
:tLAMBDA, nil, EXPR_ENDFN,
|
1637
|
+
:kDO, "do", EXPR_BEG,
|
1638
|
+
:kEND, "end", EXPR_END,
|
1639
|
+
:kDO, "do", EXPR_BEG,
|
1640
|
+
:kEND, "end", EXPR_END)
|
1603
1641
|
end
|
1604
1642
|
|
1605
|
-
def
|
1606
|
-
|
1607
|
-
|
1643
|
+
def test_yylex_lambda_hash
|
1644
|
+
assert_lex("-> (a={}) { }",
|
1645
|
+
s(:iter, s(:lambda),
|
1646
|
+
s(:args, s(:lasgn, :a, s(:hash)))),
|
1608
1647
|
|
1609
|
-
|
1610
|
-
|
1648
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1649
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1650
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1651
|
+
:tEQL, "=", EXPR_BEG, 1, 0,
|
1652
|
+
:tLBRACE, "{", EXPR_PAR, 1, 1,
|
1653
|
+
:tRCURLY, "}", EXPR_END, 1, 0,
|
1654
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1655
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1656
|
+
:tRCURLY, "}", EXPR_END, 0, 0)
|
1611
1657
|
end
|
1612
1658
|
|
1613
|
-
def
|
1614
|
-
|
1615
|
-
end
|
1659
|
+
def test_yylex_lambda_hash__24
|
1660
|
+
setup_lexer_class RubyParser::V24
|
1616
1661
|
|
1617
|
-
|
1618
|
-
|
1619
|
-
|
1662
|
+
assert_lex("-> (a={}) { }",
|
1663
|
+
s(:iter, s(:lambda),
|
1664
|
+
s(:args, s(:lasgn, :a, s(:hash)))),
|
1620
1665
|
|
1621
|
-
|
1622
|
-
|
1666
|
+
:tLAMBDA, nil, EXPR_ENDFN, 0, 0,
|
1667
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1668
|
+
:tIDENTIFIER, "a", EXPR_ARG, 1, 0,
|
1669
|
+
:tEQL, "=", EXPR_BEG, 1, 0,
|
1670
|
+
:tLBRACE, "{", EXPR_PAR, 1, 1,
|
1671
|
+
:tRCURLY, "}", EXPR_ENDARG, 1, 0,
|
1672
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1673
|
+
:tLCURLY, "{", EXPR_PAR, 0, 1,
|
1674
|
+
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
1623
1675
|
end
|
1624
1676
|
|
1625
|
-
def
|
1626
|
-
|
1627
|
-
|
1677
|
+
def test_yylex_lasgn_call_same_name
|
1678
|
+
assert_lex("a = b.c :d => 1",
|
1679
|
+
s(:lasgn, :a,
|
1680
|
+
s(:call, s(:call, nil, :b), :c,
|
1681
|
+
s(:hash, s(:lit, :d), s(:lit, 1)))),
|
1628
1682
|
|
1629
|
-
|
1630
|
-
|
1683
|
+
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
1684
|
+
:tEQL, "=", EXPR_BEG, 0, 0,
|
1685
|
+
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
1686
|
+
:tDOT, ".", EXPR_DOT, 0, 0,
|
1687
|
+
:tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
|
1688
|
+
:tSYMBOL, "d", EXPR_LIT, 0, 0,
|
1689
|
+
:tASSOC, "=>", EXPR_BEG, 0, 0,
|
1690
|
+
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
1631
1691
|
|
1632
|
-
|
1692
|
+
assert_lex("a = b.a :d => 1",
|
1693
|
+
s(:lasgn, :a,
|
1694
|
+
s(:call, s(:call, nil, :b), :a,
|
1695
|
+
s(:hash, s(:lit, :d), s(:lit, 1)))),
|
1696
|
+
|
1697
|
+
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
1698
|
+
:tEQL, "=", EXPR_BEG, 0, 0,
|
1699
|
+
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
1700
|
+
:tDOT, ".", EXPR_DOT, 0, 0,
|
1701
|
+
:tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
|
1702
|
+
:tSYMBOL, "d", EXPR_LIT, 0, 0,
|
1703
|
+
:tASSOC, "=>", EXPR_BEG, 0, 0,
|
1704
|
+
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
1633
1705
|
end
|
1634
1706
|
|
1635
1707
|
def test_yylex_lt
|
@@ -1656,12 +1728,22 @@ class TestRubyLexer < Minitest::Test
|
|
1656
1728
|
assert_lex3("<=", nil, :tLEQ, "<=", EXPR_BEG)
|
1657
1729
|
end
|
1658
1730
|
|
1731
|
+
def test_yylex_method_parens_chevron
|
1732
|
+
assert_lex("a()<<1",
|
1733
|
+
s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
|
1734
|
+
:tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
|
1735
|
+
:tLPAREN2, "(", EXPR_PAR, 1, 0,
|
1736
|
+
:tRPAREN, ")", EXPR_ENDFN, 0, 0,
|
1737
|
+
:tLSHFT, "<<", EXPR_BEG, 0, 0,
|
1738
|
+
:tINTEGER, 1, EXPR_NUM, 0, 0)
|
1739
|
+
end
|
1740
|
+
|
1659
1741
|
def test_yylex_minus
|
1660
1742
|
assert_lex3("1 - 2",
|
1661
1743
|
nil,
|
1662
|
-
:tINTEGER, 1,
|
1744
|
+
:tINTEGER, 1, EXPR_NUM,
|
1663
1745
|
:tMINUS, "-", EXPR_BEG,
|
1664
|
-
:tINTEGER, 2,
|
1746
|
+
:tINTEGER, 2, EXPR_NUM)
|
1665
1747
|
end
|
1666
1748
|
|
1667
1749
|
def test_yylex_minus_equals
|
@@ -1684,7 +1766,39 @@ class TestRubyLexer < Minitest::Test
|
|
1684
1766
|
assert_lex3("-42",
|
1685
1767
|
nil,
|
1686
1768
|
:tUMINUS_NUM, "-", EXPR_BEG,
|
1687
|
-
:tINTEGER, 42,
|
1769
|
+
:tINTEGER, 42, EXPR_NUM)
|
1770
|
+
end
|
1771
|
+
|
1772
|
+
def test_yylex_not_at_defn
|
1773
|
+
assert_lex("def +@; end",
|
1774
|
+
s(:defn, :+@, s(:args), s(:nil)),
|
1775
|
+
|
1776
|
+
:kDEF, "def", EXPR_FNAME, 0, 0,
|
1777
|
+
:tUPLUS, "+@", EXPR_ARG, 0, 0,
|
1778
|
+
:tSEMI, ";", EXPR_BEG, 0, 0,
|
1779
|
+
:kEND, "end", EXPR_END, 0, 0)
|
1780
|
+
|
1781
|
+
assert_lex("def !@; end",
|
1782
|
+
s(:defn, :"!@", s(:args), s(:nil)),
|
1783
|
+
|
1784
|
+
:kDEF, "def", EXPR_FNAME, 0, 0,
|
1785
|
+
:tUBANG, "!@", EXPR_ARG, 0, 0,
|
1786
|
+
:tSEMI, ";", EXPR_BEG, 0, 0,
|
1787
|
+
:kEND, "end", EXPR_END, 0, 0)
|
1788
|
+
end
|
1789
|
+
|
1790
|
+
def test_yylex_not_at_ivar
|
1791
|
+
assert_lex("!@ivar",
|
1792
|
+
s(:call, s(:ivar, :@ivar), :"!"),
|
1793
|
+
|
1794
|
+
:tBANG, "!", EXPR_BEG, 0, 0,
|
1795
|
+
:tIVAR, "@ivar", EXPR_END, 0, 0)
|
1796
|
+
end
|
1797
|
+
|
1798
|
+
def test_yylex_not_unary_method
|
1799
|
+
self.lex_state = EXPR_FNAME
|
1800
|
+
|
1801
|
+
assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
|
1688
1802
|
end
|
1689
1803
|
|
1690
1804
|
def test_yylex_nth_ref
|
@@ -1700,7 +1814,68 @@ class TestRubyLexer < Minitest::Test
|
|
1700
1814
|
:tNTH_REF, 7, EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
1701
1815
|
:tNTH_REF, 8, EXPR_END, :tCOMMA, ",", EXPR_PAR,
|
1702
1816
|
:tNTH_REF, 9, EXPR_END,
|
1703
|
-
:tRBRACK, "]",
|
1817
|
+
:tRBRACK, "]", EXPR_END)
|
1818
|
+
end
|
1819
|
+
|
1820
|
+
def test_yylex_number_times_ident_times_return_number
|
1821
|
+
assert_lex("1 * b * 3",
|
1822
|
+
s(:call,
|
1823
|
+
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
|
1824
|
+
:*, s(:lit, 3)),
|
1825
|
+
|
1826
|
+
:tINTEGER, 1, EXPR_NUM, 0, 0,
|
1827
|
+
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
1828
|
+
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
1829
|
+
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
1830
|
+
:tINTEGER, 3, EXPR_NUM, 0, 0)
|
1831
|
+
|
1832
|
+
assert_lex("1 * b *\n 3",
|
1833
|
+
s(:call,
|
1834
|
+
s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
|
1835
|
+
:*, s(:lit, 3)),
|
1836
|
+
|
1837
|
+
:tINTEGER, 1, EXPR_NUM, 0, 0,
|
1838
|
+
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
1839
|
+
:tIDENTIFIER, "b", EXPR_ARG, 0, 0,
|
1840
|
+
:tSTAR2, "*", EXPR_BEG, 0, 0,
|
1841
|
+
:tINTEGER, 3, EXPR_NUM, 0, 0)
|
1842
|
+
end
|
1843
|
+
|
1844
|
+
def test_yylex_numbers
|
1845
|
+
assert_lex3 "0b10", nil, :tINTEGER, 2, EXPR_NUM
|
1846
|
+
assert_lex3 "0B10", nil, :tINTEGER, 2, EXPR_NUM
|
1847
|
+
|
1848
|
+
assert_lex3 "0d10", nil, :tINTEGER, 10, EXPR_NUM
|
1849
|
+
assert_lex3 "0D10", nil, :tINTEGER, 10, EXPR_NUM
|
1850
|
+
|
1851
|
+
assert_lex3 "0x10", nil, :tINTEGER, 16, EXPR_NUM
|
1852
|
+
assert_lex3 "0X10", nil, :tINTEGER, 16, EXPR_NUM
|
1853
|
+
|
1854
|
+
assert_lex3 "0o10", nil, :tINTEGER, 8, EXPR_NUM
|
1855
|
+
assert_lex3 "0O10", nil, :tINTEGER, 8, EXPR_NUM
|
1856
|
+
|
1857
|
+
assert_lex3 "0o", nil, :tINTEGER, 0, EXPR_NUM
|
1858
|
+
assert_lex3 "0O", nil, :tINTEGER, 0, EXPR_NUM
|
1859
|
+
|
1860
|
+
assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
|
1861
|
+
|
1862
|
+
refute_lex "0x"
|
1863
|
+
refute_lex "0X"
|
1864
|
+
refute_lex "0b"
|
1865
|
+
refute_lex "0B"
|
1866
|
+
refute_lex "0d"
|
1867
|
+
refute_lex "0D"
|
1868
|
+
|
1869
|
+
refute_lex "08"
|
1870
|
+
refute_lex "09"
|
1871
|
+
refute_lex "0o8"
|
1872
|
+
refute_lex "0o9"
|
1873
|
+
refute_lex "0O8"
|
1874
|
+
refute_lex "0O9"
|
1875
|
+
|
1876
|
+
refute_lex "1_e1"
|
1877
|
+
refute_lex "1_.1"
|
1878
|
+
refute_lex "1__1"
|
1704
1879
|
end
|
1705
1880
|
|
1706
1881
|
def test_yylex_open_bracket
|
@@ -1730,9 +1905,9 @@ class TestRubyLexer < Minitest::Test
|
|
1730
1905
|
assert_lex3("m { 3 }",
|
1731
1906
|
nil,
|
1732
1907
|
:tIDENTIFIER, "m", EXPR_CMDARG,
|
1733
|
-
:tLCURLY, "{",
|
1734
|
-
:tINTEGER, 3,
|
1735
|
-
:tRCURLY, "}",
|
1908
|
+
:tLCURLY, "{", EXPR_PAR,
|
1909
|
+
:tINTEGER, 3, EXPR_NUM,
|
1910
|
+
:tRCURLY, "}", EXPR_END)
|
1736
1911
|
end
|
1737
1912
|
|
1738
1913
|
def test_yylex_open_curly_bracket_block
|
@@ -1741,8 +1916,8 @@ class TestRubyLexer < Minitest::Test
|
|
1741
1916
|
assert_lex3("{ 4 }",
|
1742
1917
|
nil,
|
1743
1918
|
:tLBRACE_ARG, "{", EXPR_BEG,
|
1744
|
-
:tINTEGER, 4,
|
1745
|
-
:tRCURLY, "}",
|
1919
|
+
:tINTEGER, 4, EXPR_NUM,
|
1920
|
+
:tRCURLY, "}", EXPR_END)
|
1746
1921
|
end
|
1747
1922
|
|
1748
1923
|
def test_yylex_open_square_bracket_arg
|
@@ -1751,32 +1926,32 @@ class TestRubyLexer < Minitest::Test
|
|
1751
1926
|
assert_lex3("m [ 3 ]",
|
1752
1927
|
nil,
|
1753
1928
|
:tIDENTIFIER, "m", EXPR_CMDARG,
|
1754
|
-
:tLBRACK, "[",
|
1755
|
-
:tINTEGER, 3,
|
1756
|
-
:tRBRACK, "]",
|
1929
|
+
:tLBRACK, "[", EXPR_PAR,
|
1930
|
+
:tINTEGER, 3, EXPR_NUM,
|
1931
|
+
:tRBRACK, "]", EXPR_END)
|
1757
1932
|
end
|
1758
1933
|
|
1759
1934
|
def test_yylex_open_square_bracket_ary
|
1760
1935
|
assert_lex3("[1, 2, 3]",
|
1761
1936
|
nil,
|
1762
1937
|
:tLBRACK, "[", EXPR_PAR,
|
1763
|
-
:tINTEGER, 1, EXPR_NUM, :tCOMMA, ",",
|
1764
|
-
:tINTEGER, 2, EXPR_NUM, :tCOMMA, ",",
|
1938
|
+
:tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
|
1939
|
+
:tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
|
1765
1940
|
:tINTEGER, 3, EXPR_NUM,
|
1766
|
-
:tRBRACK, "]",
|
1941
|
+
:tRBRACK, "]", EXPR_END)
|
1767
1942
|
end
|
1768
1943
|
|
1769
1944
|
def test_yylex_open_square_bracket_meth
|
1770
1945
|
assert_lex3("m[3]",
|
1771
1946
|
nil,
|
1772
1947
|
:tIDENTIFIER, "m", EXPR_CMDARG,
|
1773
|
-
:tLBRACK2, "[",
|
1774
|
-
:tINTEGER, 3,
|
1775
|
-
:tRBRACK, "]",
|
1948
|
+
:tLBRACK2, "[", EXPR_PAR,
|
1949
|
+
:tINTEGER, 3, EXPR_NUM,
|
1950
|
+
:tRBRACK, "]", EXPR_END)
|
1776
1951
|
end
|
1777
1952
|
|
1778
1953
|
def test_yylex_or
|
1779
|
-
assert_lex3("|", nil, :tPIPE, "|",
|
1954
|
+
assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
|
1780
1955
|
end
|
1781
1956
|
|
1782
1957
|
def test_yylex_or2
|
@@ -1791,12 +1966,85 @@ class TestRubyLexer < Minitest::Test
|
|
1791
1966
|
assert_lex3("|=", nil, :tOP_ASGN, "|", EXPR_BEG)
|
1792
1967
|
end
|
1793
1968
|
|
1969
|
+
def test_yylex_paren_string_interpolated_regexp
|
1970
|
+
setup_lexer('%( #{(/abcd/)} )',
|
1971
|
+
s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
|
1972
|
+
|
1973
|
+
assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
|
1974
|
+
assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
|
1975
|
+
assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
|
1976
|
+
|
1977
|
+
emulate_string_interpolation do
|
1978
|
+
assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
|
1979
|
+
assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
|
1980
|
+
assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
|
1981
|
+
assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
|
1982
|
+
assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
|
1983
|
+
end
|
1984
|
+
|
1985
|
+
assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
|
1986
|
+
assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
|
1987
|
+
|
1988
|
+
refute_lexeme
|
1989
|
+
end
|
1990
|
+
|
1991
|
+
def test_yylex_paren_string_parens_interpolated
|
1992
|
+
setup_lexer('%((#{b}#{d}))',
|
1993
|
+
s(:dstr,
|
1994
|
+
"(",
|
1995
|
+
s(:evstr, s(:call, nil, :b)),
|
1996
|
+
s(:evstr, s(:call, nil, :d)),
|
1997
|
+
s(:str, ")")))
|
1998
|
+
|
1999
|
+
assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
|
2000
|
+
assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
|
2001
|
+
assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
|
2002
|
+
|
2003
|
+
emulate_string_interpolation do
|
2004
|
+
assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
|
2005
|
+
end
|
2006
|
+
|
2007
|
+
assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
|
2008
|
+
|
2009
|
+
emulate_string_interpolation do
|
2010
|
+
assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
|
2011
|
+
end
|
2012
|
+
|
2013
|
+
assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
|
2014
|
+
assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
|
2015
|
+
|
2016
|
+
refute_lexeme
|
2017
|
+
end
|
2018
|
+
|
2019
|
+
def test_yylex_paren_string_parens_interpolated_regexp
|
2020
|
+
setup_lexer('%((#{(/abcd/)}))',
|
2021
|
+
s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
|
2022
|
+
|
2023
|
+
assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
|
2024
|
+
assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
|
2025
|
+
|
2026
|
+
assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
|
2027
|
+
|
2028
|
+
emulate_string_interpolation do
|
2029
|
+
assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
|
2030
|
+
assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
|
2031
|
+
assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
|
2032
|
+
assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
|
2033
|
+
assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
|
2034
|
+
end
|
2035
|
+
|
2036
|
+
assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
|
2037
|
+
assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
|
2038
|
+
|
2039
|
+
refute_lexeme
|
2040
|
+
end
|
2041
|
+
|
1794
2042
|
def test_yylex_percent
|
1795
2043
|
assert_lex3("a % 2",
|
1796
2044
|
nil,
|
1797
2045
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1798
2046
|
:tPERCENT, "%", EXPR_BEG,
|
1799
|
-
:tINTEGER, 2,
|
2047
|
+
:tINTEGER, 2, EXPR_NUM)
|
1800
2048
|
end
|
1801
2049
|
|
1802
2050
|
def test_yylex_percent_equals
|
@@ -1804,15 +2052,15 @@ class TestRubyLexer < Minitest::Test
|
|
1804
2052
|
nil,
|
1805
2053
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
1806
2054
|
:tOP_ASGN, "%", EXPR_BEG,
|
1807
|
-
:tINTEGER, 2,
|
2055
|
+
:tINTEGER, 2, EXPR_NUM)
|
1808
2056
|
end
|
1809
2057
|
|
1810
2058
|
def test_yylex_plus
|
1811
2059
|
assert_lex3("1 + 1", # TODO lex_state?
|
1812
2060
|
nil,
|
1813
|
-
:tINTEGER, 1,
|
2061
|
+
:tINTEGER, 1, EXPR_NUM,
|
1814
2062
|
:tPLUS, "+", EXPR_BEG,
|
1815
|
-
:tINTEGER, 1,
|
2063
|
+
:tINTEGER, 1, EXPR_NUM)
|
1816
2064
|
end
|
1817
2065
|
|
1818
2066
|
def test_yylex_plus_equals
|
@@ -1831,55 +2079,24 @@ class TestRubyLexer < Minitest::Test
|
|
1831
2079
|
assert_lex3("+@", nil, :tUPLUS, "+@", EXPR_ARG)
|
1832
2080
|
end
|
1833
2081
|
|
1834
|
-
def
|
1835
|
-
|
1836
|
-
|
1837
|
-
assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
|
2082
|
+
def test_yylex_plus_unary_number
|
2083
|
+
assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
|
1838
2084
|
end
|
1839
2085
|
|
1840
|
-
def
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
assert_lex3("0d10", nil, :tINTEGER, 10, EXPR_NUM)
|
1845
|
-
assert_lex3("0D10", nil, :tINTEGER, 10, EXPR_NUM)
|
1846
|
-
|
1847
|
-
assert_lex3("0x10", nil, :tINTEGER, 16, EXPR_NUM)
|
1848
|
-
assert_lex3("0X10", nil, :tINTEGER, 16, EXPR_NUM)
|
1849
|
-
|
1850
|
-
assert_lex3("0o10", nil, :tINTEGER, 8, EXPR_NUM)
|
1851
|
-
assert_lex3("0O10", nil, :tINTEGER, 8, EXPR_NUM)
|
1852
|
-
|
1853
|
-
assert_lex3("0o", nil, :tINTEGER, 0, EXPR_NUM)
|
1854
|
-
assert_lex3("0O", nil, :tINTEGER, 0, EXPR_NUM)
|
1855
|
-
|
1856
|
-
assert_lex3("0", nil, :tINTEGER, 0, EXPR_NUM)
|
1857
|
-
|
1858
|
-
refute_lex "0x"
|
1859
|
-
refute_lex "0X"
|
1860
|
-
refute_lex "0b"
|
1861
|
-
refute_lex "0B"
|
1862
|
-
refute_lex "0d"
|
1863
|
-
refute_lex "0D"
|
2086
|
+
def test_yylex_question_bad_eos
|
2087
|
+
refute_lex "?"
|
2088
|
+
end
|
1864
2089
|
|
1865
|
-
|
1866
|
-
|
1867
|
-
refute_lex "0o8"
|
1868
|
-
refute_lex "0o9"
|
1869
|
-
refute_lex "0O8"
|
1870
|
-
refute_lex "0O9"
|
2090
|
+
def test_yylex_question_eh_a__20
|
2091
|
+
setup_lexer_class RubyParser::V20
|
1871
2092
|
|
1872
|
-
|
1873
|
-
refute_lex "1_.1"
|
1874
|
-
refute_lex "1__1"
|
2093
|
+
assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
|
1875
2094
|
end
|
1876
2095
|
|
1877
|
-
def
|
1878
|
-
|
1879
|
-
end
|
2096
|
+
def test_yylex_question_eh_escape_M_escape_C__20
|
2097
|
+
setup_lexer_class RubyParser::V20
|
1880
2098
|
|
1881
|
-
|
1882
|
-
refute_lex "?"
|
2099
|
+
assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
|
1883
2100
|
end
|
1884
2101
|
|
1885
2102
|
def test_yylex_question_ws
|
@@ -1903,11 +2120,11 @@ class TestRubyLexer < Minitest::Test
|
|
1903
2120
|
end
|
1904
2121
|
|
1905
2122
|
def test_yylex_rbracket
|
1906
|
-
assert_lex3("]", nil, :tRBRACK, "]",
|
2123
|
+
assert_lex3("]", nil, :tRBRACK, "]", EXPR_END)
|
1907
2124
|
end
|
1908
2125
|
|
1909
2126
|
def test_yylex_rcurly
|
1910
|
-
assert_lex("}", nil, :tRCURLY, "}",
|
2127
|
+
assert_lex("}", nil, :tRCURLY, "}", EXPR_END, 0, 1) do
|
1911
2128
|
lexer.brace_nest += 2
|
1912
2129
|
end
|
1913
2130
|
end
|
@@ -1917,7 +2134,7 @@ class TestRubyLexer < Minitest::Test
|
|
1917
2134
|
nil,
|
1918
2135
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1919
2136
|
:tSTRING_CONTENT, "regexp", EXPR_BEG,
|
1920
|
-
:tREGEXP_END, "",
|
2137
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1921
2138
|
end
|
1922
2139
|
|
1923
2140
|
def test_yylex_regexp_ambiguous
|
@@ -1926,7 +2143,7 @@ class TestRubyLexer < Minitest::Test
|
|
1926
2143
|
:tIDENTIFIER, "method", EXPR_CMDARG,
|
1927
2144
|
:tREGEXP_BEG, "/", EXPR_CMDARG,
|
1928
2145
|
:tSTRING_CONTENT, "regexp", EXPR_CMDARG,
|
1929
|
-
:tREGEXP_END, "",
|
2146
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1930
2147
|
end
|
1931
2148
|
|
1932
2149
|
def test_yylex_regexp_bad
|
@@ -1940,7 +2157,7 @@ class TestRubyLexer < Minitest::Test
|
|
1940
2157
|
nil,
|
1941
2158
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1942
2159
|
:tSTRING_CONTENT, "regex\\C-x", EXPR_BEG,
|
1943
|
-
:tREGEXP_END, "",
|
2160
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1944
2161
|
end
|
1945
2162
|
|
1946
2163
|
def test_yylex_regexp_escape_C_M
|
@@ -1948,7 +2165,7 @@ class TestRubyLexer < Minitest::Test
|
|
1948
2165
|
nil,
|
1949
2166
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1950
2167
|
:tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
|
1951
|
-
:tREGEXP_END, "",
|
2168
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1952
2169
|
end
|
1953
2170
|
|
1954
2171
|
def test_yylex_regexp_escape_C_M_craaaazy
|
@@ -1956,7 +2173,7 @@ class TestRubyLexer < Minitest::Test
|
|
1956
2173
|
nil,
|
1957
2174
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1958
2175
|
:tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
|
1959
|
-
:tREGEXP_END, "",
|
2176
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1960
2177
|
end
|
1961
2178
|
|
1962
2179
|
def test_yylex_regexp_escape_C_bad_dash
|
@@ -1984,7 +2201,7 @@ class TestRubyLexer < Minitest::Test
|
|
1984
2201
|
nil,
|
1985
2202
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1986
2203
|
:tSTRING_CONTENT, "regex\\M-x", EXPR_BEG,
|
1987
|
-
:tREGEXP_END, "",
|
2204
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1988
2205
|
end
|
1989
2206
|
|
1990
2207
|
def test_yylex_regexp_escape_M_C
|
@@ -1992,7 +2209,7 @@ class TestRubyLexer < Minitest::Test
|
|
1992
2209
|
nil,
|
1993
2210
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
1994
2211
|
:tSTRING_CONTENT, "regex\\M-\\C-x", EXPR_BEG,
|
1995
|
-
:tREGEXP_END, "",
|
2212
|
+
:tREGEXP_END, "", EXPR_LIT)
|
1996
2213
|
end
|
1997
2214
|
|
1998
2215
|
def test_yylex_regexp_escape_M_bad_dash
|
@@ -2016,7 +2233,7 @@ class TestRubyLexer < Minitest::Test
|
|
2016
2233
|
nil,
|
2017
2234
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2018
2235
|
:tSTRING_CONTENT, "\\/", EXPR_BEG,
|
2019
|
-
:tREGEXP_END, "",
|
2236
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2020
2237
|
end
|
2021
2238
|
|
2022
2239
|
def test_yylex_regexp_escape_backslash_terminator
|
@@ -2024,15 +2241,7 @@ class TestRubyLexer < Minitest::Test
|
|
2024
2241
|
nil,
|
2025
2242
|
:tREGEXP_BEG, "%r\000", EXPR_BEG,
|
2026
2243
|
:tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
|
2027
|
-
:tREGEXP_END, "",
|
2028
|
-
end
|
2029
|
-
|
2030
|
-
def test_yylex_regexp_escaped_delim
|
2031
|
-
assert_lex3("%r!blah(?\\!blah)!",
|
2032
|
-
nil,
|
2033
|
-
:tREGEXP_BEG, "%r\000", EXPR_BEG,
|
2034
|
-
:tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
|
2035
|
-
:tREGEXP_END, "", EXPR_END)
|
2244
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2036
2245
|
end
|
2037
2246
|
|
2038
2247
|
def test_yylex_regexp_escape_backslash_terminator_meta1
|
@@ -2040,7 +2249,7 @@ class TestRubyLexer < Minitest::Test
|
|
2040
2249
|
nil,
|
2041
2250
|
:tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
|
2042
2251
|
:tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
|
2043
|
-
:tREGEXP_END, "",
|
2252
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2044
2253
|
end
|
2045
2254
|
|
2046
2255
|
def test_yylex_regexp_escape_backslash_terminator_meta2
|
@@ -2048,7 +2257,7 @@ class TestRubyLexer < Minitest::Test
|
|
2048
2257
|
nil,
|
2049
2258
|
:tREGEXP_BEG, "%r\000", EXPR_BEG,
|
2050
2259
|
:tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
|
2051
|
-
:tREGEXP_END, "",
|
2260
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2052
2261
|
end
|
2053
2262
|
|
2054
2263
|
def test_yylex_regexp_escape_backslash_terminator_meta3
|
@@ -2056,7 +2265,7 @@ class TestRubyLexer < Minitest::Test
|
|
2056
2265
|
nil,
|
2057
2266
|
:tREGEXP_BEG, "%r\000", EXPR_BEG,
|
2058
2267
|
:tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
|
2059
|
-
:tREGEXP_END, "",
|
2268
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2060
2269
|
end
|
2061
2270
|
|
2062
2271
|
def test_yylex_regexp_escape_bad_eos
|
@@ -2068,7 +2277,7 @@ class TestRubyLexer < Minitest::Test
|
|
2068
2277
|
nil,
|
2069
2278
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2070
2279
|
:tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
|
2071
|
-
:tREGEXP_END, "",
|
2280
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2072
2281
|
end
|
2073
2282
|
|
2074
2283
|
def test_yylex_regexp_escape_c
|
@@ -2076,7 +2285,7 @@ class TestRubyLexer < Minitest::Test
|
|
2076
2285
|
nil,
|
2077
2286
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2078
2287
|
:tSTRING_CONTENT, "regex\\cxxx", EXPR_BEG,
|
2079
|
-
:tREGEXP_END, "",
|
2288
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2080
2289
|
end
|
2081
2290
|
|
2082
2291
|
def test_yylex_regexp_escape_c_backslash
|
@@ -2084,7 +2293,7 @@ class TestRubyLexer < Minitest::Test
|
|
2084
2293
|
nil,
|
2085
2294
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2086
2295
|
:tSTRING_CONTENT, "regex\\c\\n", EXPR_BEG,
|
2087
|
-
:tREGEXP_END, "",
|
2296
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2088
2297
|
end
|
2089
2298
|
|
2090
2299
|
def test_yylex_regexp_escape_chars
|
@@ -2092,7 +2301,7 @@ class TestRubyLexer < Minitest::Test
|
|
2092
2301
|
nil,
|
2093
2302
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2094
2303
|
:tSTRING_CONTENT, "re\\tge\\nxp", EXPR_BEG,
|
2095
|
-
:tREGEXP_END, "",
|
2304
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2096
2305
|
end
|
2097
2306
|
|
2098
2307
|
def test_yylex_regexp_escape_double_backslash
|
@@ -2101,7 +2310,7 @@ class TestRubyLexer < Minitest::Test
|
|
2101
2310
|
nil,
|
2102
2311
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2103
2312
|
:tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
|
2104
|
-
:tREGEXP_END, "",
|
2313
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2105
2314
|
end
|
2106
2315
|
|
2107
2316
|
def test_yylex_regexp_escape_hex
|
@@ -2109,7 +2318,7 @@ class TestRubyLexer < Minitest::Test
|
|
2109
2318
|
nil,
|
2110
2319
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2111
2320
|
:tSTRING_CONTENT, "regex\\x61xp", EXPR_BEG,
|
2112
|
-
:tREGEXP_END, "",
|
2321
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2113
2322
|
end
|
2114
2323
|
|
2115
2324
|
def test_yylex_regexp_escape_hex_bad
|
@@ -2121,7 +2330,7 @@ class TestRubyLexer < Minitest::Test
|
|
2121
2330
|
nil,
|
2122
2331
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2123
2332
|
:tSTRING_CONTENT, "^[\\xd\\xa]{2}", EXPR_BEG,
|
2124
|
-
:tREGEXP_END, "on",
|
2333
|
+
:tREGEXP_END, "on", EXPR_LIT)
|
2125
2334
|
end
|
2126
2335
|
|
2127
2336
|
def test_yylex_regexp_escape_oct1
|
@@ -2129,7 +2338,7 @@ class TestRubyLexer < Minitest::Test
|
|
2129
2338
|
nil,
|
2130
2339
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2131
2340
|
:tSTRING_CONTENT, "regex\\0xp", EXPR_BEG,
|
2132
|
-
:tREGEXP_END, "",
|
2341
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2133
2342
|
end
|
2134
2343
|
|
2135
2344
|
def test_yylex_regexp_escape_oct2
|
@@ -2137,7 +2346,7 @@ class TestRubyLexer < Minitest::Test
|
|
2137
2346
|
nil,
|
2138
2347
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2139
2348
|
:tSTRING_CONTENT, "regex\\07xp", EXPR_BEG,
|
2140
|
-
:tREGEXP_END, "",
|
2349
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2141
2350
|
end
|
2142
2351
|
|
2143
2352
|
def test_yylex_regexp_escape_oct3
|
@@ -2145,15 +2354,23 @@ class TestRubyLexer < Minitest::Test
|
|
2145
2354
|
nil,
|
2146
2355
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2147
2356
|
:tSTRING_CONTENT, "regex\\10142", EXPR_BEG,
|
2148
|
-
:tREGEXP_END, "",
|
2357
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2149
2358
|
end
|
2150
2359
|
|
2151
2360
|
def test_yylex_regexp_escape_return
|
2152
2361
|
assert_lex3("/regex\\\nregex/",
|
2153
2362
|
nil,
|
2154
|
-
:tREGEXP_BEG, "/", EXPR_BEG,
|
2155
|
-
:tSTRING_CONTENT, "regexregex", EXPR_BEG,
|
2156
|
-
:tREGEXP_END, "",
|
2363
|
+
:tREGEXP_BEG, "/", EXPR_BEG,
|
2364
|
+
:tSTRING_CONTENT, "regexregex", EXPR_BEG,
|
2365
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2366
|
+
end
|
2367
|
+
|
2368
|
+
def test_yylex_regexp_escaped_delim
|
2369
|
+
assert_lex3("%r!blah(?\\!blah)!",
|
2370
|
+
nil,
|
2371
|
+
:tREGEXP_BEG, "%r\000", EXPR_BEG,
|
2372
|
+
:tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
|
2373
|
+
:tREGEXP_END, "", EXPR_LIT)
|
2157
2374
|
end
|
2158
2375
|
|
2159
2376
|
def test_yylex_regexp_nm
|
@@ -2161,7 +2378,20 @@ class TestRubyLexer < Minitest::Test
|
|
2161
2378
|
nil,
|
2162
2379
|
:tREGEXP_BEG, "/", EXPR_BEG,
|
2163
2380
|
:tSTRING_CONTENT, ".*", EXPR_BEG,
|
2164
|
-
:tREGEXP_END, "nm",
|
2381
|
+
:tREGEXP_END, "nm", EXPR_LIT)
|
2382
|
+
end
|
2383
|
+
|
2384
|
+
def test_yylex_required_kwarg_no_value_22
|
2385
|
+
setup_lexer_class RubyParser::V22
|
2386
|
+
|
2387
|
+
assert_lex3("def foo a:, b:\nend",
|
2388
|
+
nil,
|
2389
|
+
:kDEF, "def", EXPR_FNAME,
|
2390
|
+
:tIDENTIFIER, "foo", EXPR_ENDFN,
|
2391
|
+
:tLABEL, "a", EXPR_LAB,
|
2392
|
+
:tCOMMA, ",", EXPR_PAR,
|
2393
|
+
:tLABEL, "b", EXPR_LAB,
|
2394
|
+
:kEND, "end", EXPR_END)
|
2165
2395
|
end
|
2166
2396
|
|
2167
2397
|
def test_yylex_rparen
|
@@ -2173,7 +2403,7 @@ class TestRubyLexer < Minitest::Test
|
|
2173
2403
|
nil,
|
2174
2404
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
2175
2405
|
:tRSHFT, ">>", EXPR_BEG,
|
2176
|
-
:tINTEGER, 2,
|
2406
|
+
:tINTEGER, 2, EXPR_NUM)
|
2177
2407
|
end
|
2178
2408
|
|
2179
2409
|
def test_yylex_rshft_equals
|
@@ -2181,7 +2411,7 @@ class TestRubyLexer < Minitest::Test
|
|
2181
2411
|
nil,
|
2182
2412
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
2183
2413
|
:tOP_ASGN, ">>", EXPR_BEG,
|
2184
|
-
:tINTEGER, 2,
|
2414
|
+
:tINTEGER, 2, EXPR_NUM)
|
2185
2415
|
end
|
2186
2416
|
|
2187
2417
|
def test_yylex_star
|
@@ -2249,11 +2479,11 @@ class TestRubyLexer < Minitest::Test
|
|
2249
2479
|
end
|
2250
2480
|
|
2251
2481
|
def test_yylex_string_bad_eos
|
2252
|
-
refute_lex(
|
2482
|
+
refute_lex("%", :tSTRING_BEG, "%")
|
2253
2483
|
end
|
2254
2484
|
|
2255
2485
|
def test_yylex_string_bad_eos_quote
|
2256
|
-
refute_lex(
|
2486
|
+
refute_lex("%{nest", :tSTRING_BEG, "%}")
|
2257
2487
|
end
|
2258
2488
|
|
2259
2489
|
def test_yylex_string_double
|
@@ -2269,7 +2499,7 @@ class TestRubyLexer < Minitest::Test
|
|
2269
2499
|
nil,
|
2270
2500
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2271
2501
|
:tSTRING_CONTENT, "\034", EXPR_BEG,
|
2272
|
-
:tSTRING_END, "\"",
|
2502
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2273
2503
|
end
|
2274
2504
|
|
2275
2505
|
def test_yylex_string_double_escape_C_escape
|
@@ -2277,106 +2507,25 @@ class TestRubyLexer < Minitest::Test
|
|
2277
2507
|
nil,
|
2278
2508
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2279
2509
|
:tSTRING_CONTENT, "\201", EXPR_BEG,
|
2280
|
-
:tSTRING_END, "\"",
|
2510
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2281
2511
|
end
|
2282
2512
|
|
2283
2513
|
def test_yylex_string_double_escape_C_question
|
2284
2514
|
assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", EXPR_END)
|
2285
2515
|
end
|
2286
2516
|
|
2287
|
-
def test_yylex_string_utf8_simple
|
2288
|
-
chr = [0x3024].pack("U")
|
2289
|
-
|
2290
|
-
assert_lex3('"\u{3024}"',
|
2291
|
-
s(:str, chr),
|
2292
|
-
:tSTRING, chr, EXPR_END)
|
2293
|
-
end
|
2294
|
-
|
2295
|
-
def test_yylex_string_utf8_trailing_hex
|
2296
|
-
chr = [0x3024].pack("U")
|
2297
|
-
str = "#{chr}abz"
|
2298
|
-
|
2299
|
-
assert_lex3('"\u3024abz"',
|
2300
|
-
s(:str, str),
|
2301
|
-
:tSTRING, str, EXPR_END)
|
2302
|
-
end
|
2303
|
-
|
2304
|
-
def test_yylex_string_utf8_missing_hex
|
2305
|
-
refute_lex('"\u3zzz"')
|
2306
|
-
refute_lex('"\u30zzz"')
|
2307
|
-
refute_lex('"\u302zzz"')
|
2308
|
-
end
|
2309
|
-
|
2310
|
-
def test_yylex_string_utf8_complex
|
2311
|
-
chr = [0x3024].pack("U")
|
2312
|
-
|
2313
|
-
assert_lex3('"#@a\u{3024}"',
|
2314
|
-
s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
|
2315
|
-
:tSTRING_BEG, '"', EXPR_BEG,
|
2316
|
-
:tSTRING_DVAR, nil, EXPR_BEG,
|
2317
|
-
:tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
|
2318
|
-
:tSTRING_END, '"', EXPR_END)
|
2319
|
-
end
|
2320
|
-
|
2321
|
-
def test_yylex_string_utf8_complex_trailing_hex
|
2322
|
-
chr = [0x3024].pack("U")
|
2323
|
-
str = "#{chr}abz"
|
2324
|
-
|
2325
|
-
assert_lex3('"#@a\u3024abz"',
|
2326
|
-
s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
|
2327
|
-
:tSTRING_BEG, '"', EXPR_BEG,
|
2328
|
-
:tSTRING_DVAR, nil, EXPR_BEG,
|
2329
|
-
:tSTRING_CONTENT, "@a"+str, EXPR_BEG,
|
2330
|
-
:tSTRING_END, '"', EXPR_END)
|
2331
|
-
end
|
2332
|
-
|
2333
|
-
def test_yylex_string_utf8_complex_missing_hex
|
2334
|
-
chr = [0x302].pack("U")
|
2335
|
-
str = "#{chr}zzz"
|
2336
|
-
|
2337
|
-
refute_lex('"#@a\u302zzz"',
|
2338
|
-
:tSTRING_BEG, '"',
|
2339
|
-
:tSTRING_DVAR, nil,
|
2340
|
-
:tSTRING_CONTENT, "@a"+str,
|
2341
|
-
:tSTRING_END, '"')
|
2342
|
-
|
2343
|
-
chr = [0x30].pack("U")
|
2344
|
-
str = "#{chr}zzz"
|
2345
|
-
|
2346
|
-
refute_lex('"#@a\u30zzz"',
|
2347
|
-
:tSTRING_BEG, '"',
|
2348
|
-
:tSTRING_DVAR, nil,
|
2349
|
-
:tSTRING_CONTENT, "@a"+str,
|
2350
|
-
:tSTRING_END, '"')
|
2351
|
-
|
2352
|
-
chr = [0x3].pack("U")
|
2353
|
-
str = "#{chr}zzz"
|
2354
|
-
|
2355
|
-
refute_lex('"#@a\u3zzz"',
|
2356
|
-
:tSTRING_BEG, '"',
|
2357
|
-
:tSTRING_DVAR, nil,
|
2358
|
-
:tSTRING_CONTENT, "@a"+str,
|
2359
|
-
:tSTRING_END, '"')
|
2360
|
-
end
|
2361
|
-
|
2362
2517
|
def test_yylex_string_double_escape_M
|
2363
2518
|
chr = "\341"
|
2364
2519
|
|
2365
2520
|
assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, EXPR_END)
|
2366
2521
|
end
|
2367
2522
|
|
2368
|
-
def test_why_does_ruby_hate_me?
|
2369
|
-
assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
|
2370
|
-
nil,
|
2371
|
-
:tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, EXPR_END)
|
2372
|
-
end
|
2373
|
-
|
2374
2523
|
def test_yylex_string_double_escape_M_backslash
|
2375
2524
|
assert_lex3("\"\\M-\\\\\"",
|
2376
2525
|
nil,
|
2377
2526
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2378
2527
|
:tSTRING_CONTENT, "\334", EXPR_BEG,
|
2379
|
-
:tSTRING_END, "\"",
|
2528
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2380
2529
|
end
|
2381
2530
|
|
2382
2531
|
def test_yylex_string_double_escape_M_escape
|
@@ -2384,7 +2533,7 @@ class TestRubyLexer < Minitest::Test
|
|
2384
2533
|
nil,
|
2385
2534
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2386
2535
|
:tSTRING_CONTENT, "\201", EXPR_BEG,
|
2387
|
-
:tSTRING_END, "\"",
|
2536
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2388
2537
|
end
|
2389
2538
|
|
2390
2539
|
def test_yylex_string_double_escape_bs1
|
@@ -2404,7 +2553,7 @@ class TestRubyLexer < Minitest::Test
|
|
2404
2553
|
nil,
|
2405
2554
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2406
2555
|
:tSTRING_CONTENT, "\034", EXPR_BEG,
|
2407
|
-
:tSTRING_END, "\"",
|
2556
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2408
2557
|
end
|
2409
2558
|
|
2410
2559
|
def test_yylex_string_double_escape_c_escape
|
@@ -2412,7 +2561,7 @@ class TestRubyLexer < Minitest::Test
|
|
2412
2561
|
nil,
|
2413
2562
|
:tSTRING_BEG, "\"", EXPR_BEG,
|
2414
2563
|
:tSTRING_CONTENT, "\201", EXPR_BEG,
|
2415
|
-
:tSTRING_END, "\"",
|
2564
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2416
2565
|
end
|
2417
2566
|
|
2418
2567
|
def test_yylex_string_double_escape_c_question
|
@@ -2446,15 +2595,7 @@ class TestRubyLexer < Minitest::Test
|
|
2446
2595
|
:tSTRING_CONTENT, "$b c ", EXPR_BEG,
|
2447
2596
|
:tSTRING_DBEG, nil, EXPR_BEG,
|
2448
2597
|
:tSTRING_CONTENT, "3} # ", EXPR_BEG,
|
2449
|
-
:tSTRING_END, "\"",
|
2450
|
-
end
|
2451
|
-
|
2452
|
-
def test_yylex_string_double_pound_dollar_bad
|
2453
|
-
assert_lex3('"#$%"', nil,
|
2454
|
-
|
2455
|
-
:tSTRING_BEG, "\"", EXPR_BEG,
|
2456
|
-
:tSTRING_CONTENT, '#$%', EXPR_BEG,
|
2457
|
-
:tSTRING_END, "\"", EXPR_END)
|
2598
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2458
2599
|
end
|
2459
2600
|
|
2460
2601
|
def test_yylex_string_double_nested_curlies
|
@@ -2462,7 +2603,7 @@ class TestRubyLexer < Minitest::Test
|
|
2462
2603
|
nil,
|
2463
2604
|
:tSTRING_BEG, "%}", EXPR_BEG,
|
2464
2605
|
:tSTRING_CONTENT, "nest{one{two}one}nest", EXPR_BEG,
|
2465
|
-
:tSTRING_END, "}",
|
2606
|
+
:tSTRING_END, "}", EXPR_LIT)
|
2466
2607
|
end
|
2467
2608
|
|
2468
2609
|
def test_yylex_string_double_no_interp
|
@@ -2470,25 +2611,33 @@ class TestRubyLexer < Minitest::Test
|
|
2470
2611
|
assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", EXPR_END)
|
2471
2612
|
end
|
2472
2613
|
|
2614
|
+
def test_yylex_string_double_pound_dollar_bad
|
2615
|
+
assert_lex3('"#$%"', nil,
|
2616
|
+
|
2617
|
+
:tSTRING_BEG, "\"", EXPR_BEG,
|
2618
|
+
:tSTRING_CONTENT, "#\$%", EXPR_BEG,
|
2619
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2620
|
+
end
|
2621
|
+
|
2473
2622
|
def test_yylex_string_escape_x_single
|
2474
2623
|
assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", EXPR_END)
|
2475
2624
|
end
|
2476
2625
|
|
2477
|
-
def
|
2478
|
-
assert_lex3("%
|
2626
|
+
def test_yylex_string_pct_I
|
2627
|
+
assert_lex3("%I[s1 s2\ns3]",
|
2479
2628
|
nil,
|
2480
|
-
:
|
2629
|
+
:tSYMBOLS_BEG, "%I[", EXPR_BEG,
|
2481
2630
|
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
2482
2631
|
:tSPACE, nil, EXPR_BEG,
|
2483
2632
|
:tSTRING_CONTENT, "s2", EXPR_BEG,
|
2484
2633
|
:tSPACE, nil, EXPR_BEG,
|
2485
2634
|
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2486
2635
|
:tSPACE, nil, EXPR_BEG,
|
2487
|
-
:tSTRING_END, nil,
|
2636
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2488
2637
|
end
|
2489
2638
|
|
2490
|
-
def
|
2491
|
-
assert_lex3("%I[s1 s2\ns3]",
|
2639
|
+
def test_yylex_string_pct_I_extra_space
|
2640
|
+
assert_lex3("%I[ s1 s2\ns3 ]",
|
2492
2641
|
nil,
|
2493
2642
|
:tSYMBOLS_BEG, "%I[", EXPR_BEG,
|
2494
2643
|
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
@@ -2497,11 +2646,51 @@ class TestRubyLexer < Minitest::Test
|
|
2497
2646
|
:tSPACE, nil, EXPR_BEG,
|
2498
2647
|
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2499
2648
|
:tSPACE, nil, EXPR_BEG,
|
2500
|
-
:tSTRING_END, nil,
|
2649
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2501
2650
|
end
|
2502
2651
|
|
2503
|
-
def
|
2504
|
-
assert_lex3("%
|
2652
|
+
def test_yylex_string_pct_Q
|
2653
|
+
assert_lex3("%Q[s1 s2]",
|
2654
|
+
nil,
|
2655
|
+
:tSTRING_BEG, "%Q[", EXPR_BEG,
|
2656
|
+
:tSTRING_CONTENT, "s1 s2", EXPR_BEG,
|
2657
|
+
:tSTRING_END, "]", EXPR_LIT)
|
2658
|
+
end
|
2659
|
+
|
2660
|
+
def test_yylex_string_pct_W
|
2661
|
+
assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
|
2662
|
+
nil,
|
2663
|
+
:tWORDS_BEG, "%W[", EXPR_BEG,
|
2664
|
+
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
2665
|
+
:tSPACE, nil, EXPR_BEG,
|
2666
|
+
:tSTRING_CONTENT, "s2", EXPR_BEG,
|
2667
|
+
:tSPACE, nil, EXPR_BEG,
|
2668
|
+
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2669
|
+
:tSPACE, nil, EXPR_BEG,
|
2670
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2671
|
+
end
|
2672
|
+
|
2673
|
+
def test_yylex_string_pct_W_bs_nl
|
2674
|
+
assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
|
2675
|
+
nil,
|
2676
|
+
:tWORDS_BEG, "%W[", EXPR_BEG,
|
2677
|
+
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
2678
|
+
:tSPACE, nil, EXPR_BEG,
|
2679
|
+
:tSTRING_CONTENT, "\ns2", EXPR_BEG,
|
2680
|
+
:tSPACE, nil, EXPR_BEG,
|
2681
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2682
|
+
end
|
2683
|
+
|
2684
|
+
def test_yylex_string_pct_angle
|
2685
|
+
assert_lex3("%<blah>",
|
2686
|
+
nil,
|
2687
|
+
:tSTRING_BEG, "%>", EXPR_BEG,
|
2688
|
+
:tSTRING_CONTENT, "blah", EXPR_BEG,
|
2689
|
+
:tSTRING_END, ">", EXPR_LIT)
|
2690
|
+
end
|
2691
|
+
|
2692
|
+
def test_yylex_string_pct_i
|
2693
|
+
assert_lex3("%i[s1 s2\ns3]",
|
2505
2694
|
nil,
|
2506
2695
|
:tQSYMBOLS_BEG, "%i[", EXPR_BEG,
|
2507
2696
|
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
@@ -2510,36 +2699,36 @@ class TestRubyLexer < Minitest::Test
|
|
2510
2699
|
:tSPACE, nil, EXPR_BEG,
|
2511
2700
|
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2512
2701
|
:tSPACE, nil, EXPR_BEG,
|
2513
|
-
:tSTRING_END, nil,
|
2702
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2514
2703
|
end
|
2515
2704
|
|
2516
|
-
def
|
2517
|
-
assert_lex3("%
|
2705
|
+
def test_yylex_string_pct_i_extra_space
|
2706
|
+
assert_lex3("%i[ s1 s2\ns3 ]",
|
2518
2707
|
nil,
|
2519
|
-
:
|
2708
|
+
:tQSYMBOLS_BEG, "%i[", EXPR_BEG,
|
2520
2709
|
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
2521
2710
|
:tSPACE, nil, EXPR_BEG,
|
2522
2711
|
:tSTRING_CONTENT, "s2", EXPR_BEG,
|
2523
2712
|
:tSPACE, nil, EXPR_BEG,
|
2524
2713
|
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2525
2714
|
:tSPACE, nil, EXPR_BEG,
|
2526
|
-
:tSTRING_END, nil,
|
2715
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2527
2716
|
end
|
2528
2717
|
|
2529
|
-
def
|
2530
|
-
assert_lex3("%
|
2718
|
+
def test_yylex_string_pct_other
|
2719
|
+
assert_lex3("%%blah%",
|
2531
2720
|
nil,
|
2532
|
-
:tSTRING_BEG, "
|
2533
|
-
:tSTRING_CONTENT, "
|
2534
|
-
:tSTRING_END, "
|
2721
|
+
:tSTRING_BEG, "%%", EXPR_BEG,
|
2722
|
+
:tSTRING_CONTENT, "blah", EXPR_BEG,
|
2723
|
+
:tSTRING_END, "%", EXPR_LIT)
|
2535
2724
|
end
|
2536
2725
|
|
2537
|
-
def
|
2538
|
-
assert_lex3("%
|
2726
|
+
def test_yylex_string_pct_q
|
2727
|
+
assert_lex3("%q[s1 s2]",
|
2539
2728
|
nil,
|
2540
|
-
:tSTRING_BEG, "%
|
2729
|
+
:tSTRING_BEG, "%q[", EXPR_BEG,
|
2541
2730
|
:tSTRING_CONTENT, "s1 s2", EXPR_BEG,
|
2542
|
-
:tSTRING_END, "]",
|
2731
|
+
:tSTRING_END, "]", EXPR_LIT)
|
2543
2732
|
end
|
2544
2733
|
|
2545
2734
|
def test_yylex_string_pct_s
|
@@ -2547,109 +2736,151 @@ class TestRubyLexer < Minitest::Test
|
|
2547
2736
|
nil,
|
2548
2737
|
:tSYMBEG, "%s[", EXPR_FNAME, # TODO: :tSYM_BEG ?
|
2549
2738
|
:tSTRING_CONTENT, "s1 s2", EXPR_FNAME, # man... I don't like this
|
2550
|
-
:tSTRING_END, "]",
|
2739
|
+
:tSTRING_END, "]", EXPR_LIT)
|
2551
2740
|
end
|
2552
2741
|
|
2553
|
-
def
|
2554
|
-
|
2555
|
-
|
2556
|
-
|
2557
|
-
|
2558
|
-
|
2559
|
-
|
2560
|
-
:tSPACE, nil, EXPR_BEG,
|
2561
|
-
:tSTRING_CONTENT, "s3", EXPR_BEG,
|
2562
|
-
:tSPACE, nil, EXPR_BEG,
|
2563
|
-
:tSTRING_END, nil, EXPR_END)
|
2742
|
+
def test_yylex_string_pct_w
|
2743
|
+
refute_lex("%w[s1 s2 ",
|
2744
|
+
:tQWORDS_BEG, "%w[",
|
2745
|
+
:tSTRING_CONTENT, "s1",
|
2746
|
+
:tSPACE, nil,
|
2747
|
+
:tSTRING_CONTENT, "s2",
|
2748
|
+
:tSPACE, nil)
|
2564
2749
|
end
|
2565
2750
|
|
2566
|
-
def
|
2567
|
-
assert_lex3("%
|
2751
|
+
def test_yylex_string_pct_w_bs_nl
|
2752
|
+
assert_lex3("%w[s1 \\\ns2]",
|
2568
2753
|
nil,
|
2569
|
-
:
|
2754
|
+
:tQWORDS_BEG, "%w[", EXPR_BEG,
|
2570
2755
|
:tSTRING_CONTENT, "s1", EXPR_BEG,
|
2571
2756
|
:tSPACE, nil, EXPR_BEG,
|
2572
2757
|
:tSTRING_CONTENT, "\ns2", EXPR_BEG,
|
2573
2758
|
:tSPACE, nil, EXPR_BEG,
|
2574
|
-
:tSTRING_END, nil,
|
2759
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2760
|
+
end
|
2761
|
+
|
2762
|
+
def test_yylex_string_pct_w_bs_sp
|
2763
|
+
assert_lex3("%w[s\\ 1 s\\ 2]",
|
2764
|
+
nil,
|
2765
|
+
:tQWORDS_BEG, "%w[", EXPR_BEG,
|
2766
|
+
:tSTRING_CONTENT, "s 1", EXPR_BEG,
|
2767
|
+
:tSPACE, nil, EXPR_BEG,
|
2768
|
+
:tSTRING_CONTENT, "s 2", EXPR_BEG,
|
2769
|
+
:tSPACE, nil, EXPR_BEG,
|
2770
|
+
:tSTRING_END, nil, EXPR_LIT)
|
2771
|
+
end
|
2772
|
+
|
2773
|
+
def test_yylex_string_single
|
2774
|
+
assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
|
2775
|
+
end
|
2776
|
+
|
2777
|
+
def test_yylex_string_single_escape_chars
|
2778
|
+
assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
|
2779
|
+
end
|
2780
|
+
|
2781
|
+
def test_yylex_string_single_escape_quote_and_backslash
|
2782
|
+
assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz",
|
2783
|
+
EXPR_LIT)
|
2784
|
+
end
|
2785
|
+
|
2786
|
+
def test_yylex_string_single_escaped_quote
|
2787
|
+
assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
|
2788
|
+
end
|
2789
|
+
|
2790
|
+
def test_yylex_string_single_nl
|
2791
|
+
assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
|
2792
|
+
end
|
2793
|
+
|
2794
|
+
def test_yylex_string_utf8_complex
|
2795
|
+
chr = [0x3024].pack("U")
|
2796
|
+
|
2797
|
+
assert_lex3('"#@a\u{3024}"',
|
2798
|
+
s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
|
2799
|
+
:tSTRING_BEG, '"', EXPR_BEG,
|
2800
|
+
:tSTRING_DVAR, nil, EXPR_BEG,
|
2801
|
+
:tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
|
2802
|
+
:tSTRING_END, '"', EXPR_LIT)
|
2575
2803
|
end
|
2576
2804
|
|
2577
|
-
def
|
2578
|
-
|
2579
|
-
|
2580
|
-
|
2581
|
-
|
2582
|
-
:
|
2583
|
-
|
2805
|
+
def test_yylex_string_utf8_complex_missing_hex
|
2806
|
+
chr = [0x302].pack("U")
|
2807
|
+
str = "#{chr}zzz"
|
2808
|
+
|
2809
|
+
refute_lex('"#@a\u302zzz"',
|
2810
|
+
:tSTRING_BEG, '"',
|
2811
|
+
:tSTRING_DVAR, nil,
|
2812
|
+
:tSTRING_CONTENT, "@a"+str,
|
2813
|
+
:tSTRING_END, '"')
|
2814
|
+
|
2815
|
+
chr = [0x30].pack("U")
|
2816
|
+
str = "#{chr}zzz"
|
2817
|
+
|
2818
|
+
refute_lex('"#@a\u30zzz"',
|
2819
|
+
:tSTRING_BEG, '"',
|
2820
|
+
:tSTRING_DVAR, nil,
|
2821
|
+
:tSTRING_CONTENT, "@a"+str,
|
2822
|
+
:tSTRING_END, '"')
|
2823
|
+
|
2824
|
+
chr = [0x3].pack("U")
|
2825
|
+
str = "#{chr}zzz"
|
2584
2826
|
|
2585
|
-
|
2586
|
-
|
2587
|
-
nil,
|
2588
|
-
:
|
2589
|
-
:
|
2590
|
-
:tSTRING_END, "%", EXPR_END)
|
2827
|
+
refute_lex('"#@a\u3zzz"',
|
2828
|
+
:tSTRING_BEG, '"',
|
2829
|
+
:tSTRING_DVAR, nil,
|
2830
|
+
:tSTRING_CONTENT, "@a"+str,
|
2831
|
+
:tSTRING_END, '"')
|
2591
2832
|
end
|
2592
2833
|
|
2593
|
-
def
|
2594
|
-
|
2595
|
-
|
2596
|
-
:tSTRING_CONTENT, "s1",
|
2597
|
-
:tSPACE, nil,
|
2598
|
-
:tSTRING_CONTENT, "s2",
|
2599
|
-
:tSPACE, nil)
|
2600
|
-
end
|
2834
|
+
def test_yylex_string_utf8_complex_trailing_hex
|
2835
|
+
chr = [0x3024].pack("U")
|
2836
|
+
str = "#{chr}abz"
|
2601
2837
|
|
2602
|
-
|
2603
|
-
|
2604
|
-
|
2605
|
-
:
|
2606
|
-
:tSTRING_CONTENT, "
|
2607
|
-
:
|
2608
|
-
:tSTRING_CONTENT, "\ns2", EXPR_BEG,
|
2609
|
-
:tSPACE, nil, EXPR_BEG,
|
2610
|
-
:tSTRING_END, nil, EXPR_END)
|
2838
|
+
assert_lex3('"#@a\u3024abz"',
|
2839
|
+
s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
|
2840
|
+
:tSTRING_BEG, '"', EXPR_BEG,
|
2841
|
+
:tSTRING_DVAR, nil, EXPR_BEG,
|
2842
|
+
:tSTRING_CONTENT, "@a"+str, EXPR_BEG,
|
2843
|
+
:tSTRING_END, '"', EXPR_LIT)
|
2611
2844
|
end
|
2612
2845
|
|
2613
|
-
def
|
2614
|
-
|
2615
|
-
|
2616
|
-
|
2617
|
-
:tSTRING_CONTENT, "s 1", EXPR_BEG,
|
2618
|
-
:tSPACE, nil, EXPR_BEG,
|
2619
|
-
:tSTRING_CONTENT, "s 2", EXPR_BEG,
|
2620
|
-
:tSPACE, nil, EXPR_BEG,
|
2621
|
-
:tSTRING_END, nil, EXPR_END)
|
2846
|
+
def test_yylex_string_utf8_missing_hex
|
2847
|
+
refute_lex('"\u3zzz"')
|
2848
|
+
refute_lex('"\u30zzz"')
|
2849
|
+
refute_lex('"\u302zzz"')
|
2622
2850
|
end
|
2623
2851
|
|
2624
|
-
def
|
2625
|
-
|
2626
|
-
end
|
2852
|
+
def test_yylex_string_utf8_simple
|
2853
|
+
chr = [0x3024].pack("U")
|
2627
2854
|
|
2628
|
-
|
2629
|
-
|
2855
|
+
assert_lex3('"\u{3024}"',
|
2856
|
+
s(:str, chr),
|
2857
|
+
:tSTRING, chr, EXPR_END)
|
2630
2858
|
end
|
2631
2859
|
|
2632
|
-
def
|
2633
|
-
|
2634
|
-
|
2860
|
+
def test_yylex_string_utf8_trailing_hex
|
2861
|
+
chr = [0x3024].pack("U")
|
2862
|
+
str = "#{chr}abz"
|
2635
2863
|
|
2636
|
-
|
2637
|
-
|
2864
|
+
assert_lex3('"\u3024abz"',
|
2865
|
+
s(:str, str),
|
2866
|
+
:tSTRING, str, EXPR_END)
|
2638
2867
|
end
|
2639
2868
|
|
2640
|
-
def
|
2641
|
-
|
2869
|
+
def test_yylex_sym_quoted
|
2870
|
+
assert_lex(":'a'",
|
2871
|
+
s(:lit, :a),
|
2872
|
+
|
2873
|
+
:tSYMBOL, "a", EXPR_LIT, 0, 0)
|
2642
2874
|
end
|
2643
2875
|
|
2644
|
-
def
|
2645
|
-
|
2646
|
-
:tSYMBOL, "symbol\0", EXPR_END)
|
2876
|
+
def test_yylex_symbol
|
2877
|
+
assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_LIT)
|
2647
2878
|
end
|
2648
2879
|
|
2649
2880
|
def test_yylex_symbol_double
|
2650
2881
|
assert_lex3(":\"symbol\"",
|
2651
2882
|
nil,
|
2652
|
-
:tSYMBOL,
|
2883
|
+
:tSYMBOL, "symbol", EXPR_LIT)
|
2653
2884
|
end
|
2654
2885
|
|
2655
2886
|
def test_yylex_symbol_double_interp
|
@@ -2659,29 +2890,30 @@ class TestRubyLexer < Minitest::Test
|
|
2659
2890
|
:tSTRING_CONTENT, "symbol", EXPR_FNAME,
|
2660
2891
|
:tSTRING_DBEG, nil, EXPR_FNAME,
|
2661
2892
|
:tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
|
2662
|
-
:tSTRING_END, "\"",
|
2893
|
+
:tSTRING_END, "\"", EXPR_LIT)
|
2663
2894
|
end
|
2664
2895
|
|
2665
2896
|
def test_yylex_symbol_single
|
2666
2897
|
assert_lex3(":'symbol'",
|
2667
2898
|
nil,
|
2668
|
-
:tSYMBOL,
|
2899
|
+
:tSYMBOL, "symbol", EXPR_LIT)
|
2669
2900
|
end
|
2670
2901
|
|
2671
|
-
def
|
2672
|
-
assert_lex3('
|
2902
|
+
def test_yylex_symbol_single_escape_chars
|
2903
|
+
assert_lex3(":'s\\tri\\ng'",
|
2673
2904
|
nil,
|
2674
|
-
:tSYMBOL,
|
2905
|
+
:tSYMBOL, "s\\tri\\ng", EXPR_LIT)
|
2675
2906
|
end
|
2676
2907
|
|
2677
|
-
def
|
2678
|
-
assert_lex3(
|
2908
|
+
def test_yylex_symbol_single_noninterp
|
2909
|
+
assert_lex3(':\'symbol#{1+1}\'',
|
2679
2910
|
nil,
|
2680
|
-
:tSYMBOL,
|
2911
|
+
:tSYMBOL, 'symbol#{1+1}', EXPR_LIT)
|
2681
2912
|
end
|
2682
2913
|
|
2683
|
-
def
|
2684
|
-
|
2914
|
+
def test_yylex_symbol_zero_byte
|
2915
|
+
assert_lex(":\"symbol\0\"", nil,
|
2916
|
+
:tSYMBOL, "symbol\0", EXPR_LIT)
|
2685
2917
|
end
|
2686
2918
|
|
2687
2919
|
def test_yylex_ternary1
|
@@ -2703,7 +2935,7 @@ class TestRubyLexer < Minitest::Test
|
|
2703
2935
|
|
2704
2936
|
assert_lex3("42 ?",
|
2705
2937
|
nil,
|
2706
|
-
:tINTEGER, 42,
|
2938
|
+
:tINTEGER, 42, EXPR_NUM,
|
2707
2939
|
:tEH, "?", EXPR_BEG)
|
2708
2940
|
end
|
2709
2941
|
|
@@ -2746,14 +2978,14 @@ class TestRubyLexer < Minitest::Test
|
|
2746
2978
|
nil,
|
2747
2979
|
:kDEF, "def", EXPR_FNAME,
|
2748
2980
|
:tIDENTIFIER, "initialize", EXPR_ENDFN,
|
2749
|
-
:tLPAREN2, "(",
|
2981
|
+
:tLPAREN2, "(", EXPR_PAR,
|
2750
2982
|
:tIDENTIFIER, "u", EXPR_ARG,
|
2751
2983
|
:tEQL, "=", EXPR_BEG,
|
2752
|
-
:tFLOAT, 0.0,
|
2753
|
-
:tCOMMA, ",",
|
2984
|
+
:tFLOAT, 0.0, EXPR_NUM,
|
2985
|
+
:tCOMMA, ",", EXPR_PAR,
|
2754
2986
|
:tIDENTIFIER, "s", EXPR_ARG,
|
2755
2987
|
:tEQL, "=", EXPR_BEG,
|
2756
|
-
:tFLOAT, 0.0,
|
2988
|
+
:tFLOAT, 0.0, EXPR_NUM)
|
2757
2989
|
end
|
2758
2990
|
|
2759
2991
|
def test_zbug_id_equals
|
@@ -2761,7 +2993,7 @@ class TestRubyLexer < Minitest::Test
|
|
2761
2993
|
nil,
|
2762
2994
|
:tIDENTIFIER, "a", EXPR_CMDARG,
|
2763
2995
|
:tEQL, "=", EXPR_BEG,
|
2764
|
-
:tFLOAT, 0.0,
|
2996
|
+
:tFLOAT, 0.0, EXPR_NUM)
|
2765
2997
|
end
|
2766
2998
|
|
2767
2999
|
def test_zbug_no_spaces_in_decl
|
@@ -2769,190 +3001,13 @@ class TestRubyLexer < Minitest::Test
|
|
2769
3001
|
nil,
|
2770
3002
|
:kDEF, "def", EXPR_FNAME,
|
2771
3003
|
:tIDENTIFIER, "initialize", EXPR_ENDFN,
|
2772
|
-
:tLPAREN2, "(",
|
3004
|
+
:tLPAREN2, "(", EXPR_PAR,
|
2773
3005
|
:tIDENTIFIER, "u", EXPR_ARG,
|
2774
3006
|
:tEQL, "=", EXPR_BEG,
|
2775
|
-
:tFLOAT, 0.0,
|
2776
|
-
:tCOMMA, ",",
|
3007
|
+
:tFLOAT, 0.0, EXPR_NUM,
|
3008
|
+
:tCOMMA, ",", EXPR_PAR,
|
2777
3009
|
:tIDENTIFIER, "s", EXPR_ARG,
|
2778
3010
|
:tEQL, "=", EXPR_BEG,
|
2779
|
-
:tFLOAT, 0.0,
|
2780
|
-
end
|
2781
|
-
|
2782
|
-
def test_pct_w_backslashes
|
2783
|
-
["\t", "\n", "\r", "\v", "\f"].each do |char|
|
2784
|
-
next if !RubyLexer::HAS_ENC and char == "\v"
|
2785
|
-
|
2786
|
-
assert_lex("%w[foo#{char}bar]",
|
2787
|
-
s(:array, s(:str, "foo"), s(:str, "bar")),
|
2788
|
-
|
2789
|
-
:tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
|
2790
|
-
:tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
|
2791
|
-
:tSPACE, nil, EXPR_BEG, 0, 0,
|
2792
|
-
:tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
|
2793
|
-
:tSPACE, nil, EXPR_BEG, 0, 0,
|
2794
|
-
:tSTRING_END, nil, EXPR_END, 0, 0)
|
2795
|
-
end
|
2796
|
-
end
|
2797
|
-
|
2798
|
-
def test_yylex_sym_quoted
|
2799
|
-
assert_lex(":'a'",
|
2800
|
-
s(:lit, :a),
|
2801
|
-
|
2802
|
-
:tSYMBOL, "a", EXPR_END, 0, 0)
|
2803
|
-
end
|
2804
|
-
|
2805
|
-
def test_yylex_hash_colon
|
2806
|
-
assert_lex("{a:1}",
|
2807
|
-
s(:hash, s(:lit, :a), s(:lit, 1)),
|
2808
|
-
|
2809
|
-
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
2810
|
-
:tLABEL, "a", EXPR_LAB, 0, 1,
|
2811
|
-
:tINTEGER, 1, EXPR_NUM, 0, 1,
|
2812
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
2813
|
-
end
|
2814
|
-
|
2815
|
-
def test_yylex_hash_colon_quoted_22
|
2816
|
-
setup_lexer_class RubyParser::V22
|
2817
|
-
|
2818
|
-
assert_lex("{'a':1}",
|
2819
|
-
s(:hash, s(:lit, :a), s(:lit, 1)),
|
2820
|
-
|
2821
|
-
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
2822
|
-
:tLABEL, "a", EXPR_LAB, 0, 1,
|
2823
|
-
:tINTEGER, 1, EXPR_NUM, 0, 1,
|
2824
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
2825
|
-
end
|
2826
|
-
|
2827
|
-
def test_yylex_hash_colon_quoted_symbol_22
|
2828
|
-
setup_lexer_class RubyParser::V22
|
2829
|
-
|
2830
|
-
assert_lex("{'abc': :b}",
|
2831
|
-
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
2832
|
-
|
2833
|
-
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
2834
|
-
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
2835
|
-
:tSYMBOL, "b", EXPR_END, 0, 1,
|
2836
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
2837
|
-
end
|
2838
|
-
|
2839
|
-
def test_yylex_hash_colon_double_quoted_symbol_22
|
2840
|
-
setup_lexer_class RubyParser::V22
|
2841
|
-
|
2842
|
-
assert_lex('{"abc": :b}',
|
2843
|
-
s(:hash, s(:lit, :abc), s(:lit, :b)),
|
2844
|
-
|
2845
|
-
:tLBRACE, "{", EXPR_PAR, 0, 1,
|
2846
|
-
:tLABEL, "abc", EXPR_LAB, 0, 1,
|
2847
|
-
:tSYMBOL, "b", EXPR_END, 0, 1,
|
2848
|
-
:tRCURLY, "}", EXPR_ENDARG, 0, 0)
|
2849
|
-
end
|
2850
|
-
|
2851
|
-
def test_yylex_required_kwarg_no_value_22
|
2852
|
-
setup_lexer_class RubyParser::V22
|
2853
|
-
|
2854
|
-
assert_lex3("def foo a:, b:\nend",
|
2855
|
-
nil,
|
2856
|
-
:kDEF, "def", EXPR_FNAME,
|
2857
|
-
:tIDENTIFIER, "foo", EXPR_ENDFN,
|
2858
|
-
:tLABEL, "a", EXPR_LAB,
|
2859
|
-
:tCOMMA, ",", EXPR_PAR,
|
2860
|
-
:tLABEL, "b", EXPR_LAB,
|
2861
|
-
:kEND, "end", EXPR_END)
|
2862
|
-
end
|
2863
|
-
|
2864
|
-
def test_yylex_hash_colon_double_quoted_with_escapes
|
2865
|
-
setup_lexer_class RubyParser::V22
|
2866
|
-
|
2867
|
-
assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
|
2868
|
-
nil,
|
2869
|
-
|
2870
|
-
:tLBRACE, "{", EXPR_PAR,
|
2871
|
-
:tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
|
2872
|
-
:tINTEGER, 1, EXPR_NUM,
|
2873
|
-
:tRCURLY, "}", EXPR_ENDARG)
|
2874
|
-
end
|
2875
|
-
|
2876
|
-
def test_yylex_hash_colon_quoted_with_escapes
|
2877
|
-
setup_lexer_class RubyParser::V22
|
2878
|
-
|
2879
|
-
assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
|
2880
|
-
nil,
|
2881
|
-
|
2882
|
-
:tLBRACE, "{", EXPR_PAR,
|
2883
|
-
:tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
|
2884
|
-
:tINTEGER, 1, EXPR_NUM,
|
2885
|
-
:tRCURLY, "}", EXPR_ENDARG)
|
2886
|
-
end
|
2887
|
-
|
2888
|
-
def test_ruby21_rational_literal
|
2889
|
-
setup_lexer_class RubyParser::V21
|
2890
|
-
|
2891
|
-
assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
|
2892
|
-
assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
|
2893
|
-
assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
|
2894
|
-
assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
|
2895
|
-
assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
|
2896
|
-
assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
|
2897
|
-
assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
|
2898
|
-
assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
|
2899
|
-
assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
|
2900
|
-
assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
|
2901
|
-
|
2902
|
-
assert_lex3("-10r", nil,
|
2903
|
-
:tUMINUS_NUM, "-", EXPR_BEG,
|
2904
|
-
:tRATIONAL, Rational(10), EXPR_NUM)
|
2905
|
-
end
|
2906
|
-
|
2907
|
-
def test_ruby21_imaginary_literal
|
2908
|
-
setup_lexer_class RubyParser::V21
|
2909
|
-
|
2910
|
-
assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
|
2911
|
-
assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
|
2912
|
-
assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
|
2913
|
-
assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
|
2914
|
-
assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
|
2915
|
-
assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
|
2916
|
-
assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
|
2917
|
-
assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
|
2918
|
-
assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
|
2919
|
-
assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
|
2920
|
-
|
2921
|
-
assert_lex3("-10i", nil,
|
2922
|
-
:tUMINUS_NUM, "-", EXPR_BEG,
|
2923
|
-
:tIMAGINARY, Complex(0, 10), EXPR_NUM)
|
2924
|
-
end
|
2925
|
-
|
2926
|
-
def test_ruby21_rational_imaginary_literal
|
2927
|
-
setup_lexer_class RubyParser::V21
|
2928
|
-
|
2929
|
-
assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
|
2930
|
-
assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
|
2931
|
-
assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
|
2932
|
-
assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
|
2933
|
-
assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
|
2934
|
-
assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
|
2935
|
-
assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
|
2936
|
-
assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
|
2937
|
-
assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
|
2938
|
-
assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
|
2939
|
-
|
2940
|
-
assert_lex3("-10ri", nil,
|
2941
|
-
:tUMINUS_NUM, "-", EXPR_BEG,
|
2942
|
-
:tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
|
2943
|
-
end
|
2944
|
-
|
2945
|
-
def test_ruby21_imaginary_literal_with_succeeding_keyword
|
2946
|
-
setup_lexer_class RubyParser::V21
|
2947
|
-
|
2948
|
-
# 2/4 scenarios are syntax errors on all tested versions so I
|
2949
|
-
# deleted them.
|
2950
|
-
|
2951
|
-
assert_lex3("1if", nil,
|
2952
|
-
:tINTEGER, 1, EXPR_NUM,
|
2953
|
-
:kIF_MOD, "if", EXPR_PAR)
|
2954
|
-
assert_lex3("1.0if", nil,
|
2955
|
-
:tFLOAT, 1.0, EXPR_NUM,
|
2956
|
-
:kIF_MOD, "if", EXPR_PAR)
|
3011
|
+
:tFLOAT, 0.0, EXPR_NUM)
|
2957
3012
|
end
|
2958
3013
|
end
|