ruby_parser 3.13.1 → 3.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  # encoding: US-ASCII
2
+
2
3
  # TODO: work this out
3
4
 
4
5
  require "minitest/autorun"
@@ -10,8 +11,8 @@ class TestRubyLexer < Minitest::Test
10
11
 
11
12
  attr_accessor :processor, :lex, :parser_class, :lex_state
12
13
 
13
- alias :lexer :lex # lets me copy/paste code from parser
14
- alias :lexer= :lex=
14
+ alias lexer lex # lets me copy/paste code from parser
15
+ alias lexer= lex=
15
16
 
16
17
  def setup
17
18
  self.lex_state = EXPR_BEG
@@ -21,12 +22,7 @@ class TestRubyLexer < Minitest::Test
21
22
  def setup_lexer input, exp_sexp = nil
22
23
  setup_new_parser
23
24
  lex.ss = RPStringScanner.new(input)
24
- lex.lex_state = self.lex_state
25
- end
26
-
27
- def setup_new_parser
28
- self.processor = parser_class.new
29
- self.lex = processor.lexer
25
+ lex.lex_state = lex_state
30
26
  end
31
27
 
32
28
  def setup_lexer_class parser_class
@@ -35,11 +31,16 @@ class TestRubyLexer < Minitest::Test
35
31
  setup_lexer "blah blah"
36
32
  end
37
33
 
38
- def assert_lex input, exp_sexp, *args, &b
34
+ def setup_new_parser
35
+ self.processor = parser_class.new
36
+ self.lex = processor.lexer
37
+ end
38
+
39
+ def assert_lex input, exp_sexp, *args
39
40
  setup_lexer input
40
41
  assert_parse input, exp_sexp if exp_sexp
41
42
 
42
- b.call if b
43
+ yield if block_given?
43
44
 
44
45
  args.each_slice(5) do |token, value, state, paren, brace|
45
46
  assert_next_lexeme token, value, state, paren, brace
@@ -54,14 +55,6 @@ class TestRubyLexer < Minitest::Test
54
55
  assert_lex(input, exp_sexp, *args, &block)
55
56
  end
56
57
 
57
- def refute_lex input, *args # TODO: re-sort
58
- args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
59
-
60
- assert_raises RubyParser::SyntaxError do
61
- assert_lex(input, nil, *args)
62
- end
63
- end
64
-
65
58
  def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
66
59
  assert_lex3("def #{name} ",
67
60
  nil,
@@ -73,27 +66,26 @@ class TestRubyLexer < Minitest::Test
73
66
  def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
74
67
  adv = @lex.next_token
75
68
 
76
- assert adv, "no more tokens"
69
+ assert adv, "no more tokens, expecting: %p %p %p %p %p" % [token, value, state, paren, brace]
77
70
 
78
71
  act_token, act_value = adv
79
72
 
80
73
  msg = message {
81
- act = [act_token, act_value, @lex.lex_state,
82
- @lex.paren_nest, @lex.brace_nest]
74
+ act = [act_token, act_value, @lex.lex_state, @lex.paren_nest, @lex.brace_nest]
83
75
  exp = [token, value, state, paren, brace]
84
76
  "#{exp.inspect} vs #{act.inspect}"
85
77
  }
86
78
 
87
79
  act_value = act_value.first if Array === act_value
88
80
 
89
- assert_equal token, act_token, msg
81
+ assert_equal token, act_token, msg
90
82
  case value
91
83
  when Float then
92
84
  assert_in_epsilon value, act_value, 0.001, msg
93
85
  when NilClass then
94
86
  assert_nil act_value, msg
95
87
  else
96
- assert_equal value, act_value, msg
88
+ assert_equal value, act_value, msg
97
89
  end
98
90
  assert_equal state, @lex.lex_state, msg if state
99
91
  assert_equal paren, @lex.paren_nest, msg if paren
@@ -116,6 +108,20 @@ class TestRubyLexer < Minitest::Test
116
108
  end
117
109
  end
118
110
 
111
+ def refute_lex input, *args # TODO: re-sort
112
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
113
+
114
+ assert_raises RubyParser::SyntaxError do
115
+ assert_lex(input, nil, *args)
116
+ end
117
+ end
118
+
119
+ def refute_lex5 input, *args
120
+ assert_raises RubyParser::SyntaxError do
121
+ assert_lex(input, *args)
122
+ end
123
+ end
124
+
119
125
  def refute_lexeme
120
126
  x = y = @lex.next_token
121
127
 
@@ -140,7 +146,7 @@ class TestRubyLexer < Minitest::Test
140
146
  yield
141
147
 
142
148
  lexer.lex_state = EXPR_ENDARG
143
- assert_next_lexeme :tSTRING_DEND, "}", EXPR_ENDARG, 0
149
+ assert_next_lexeme :tSTRING_DEND, "}", EXPR_END, 0
144
150
 
145
151
  lexer.lex_strterm = lex_strterm
146
152
  lexer.lex_state = EXPR_BEG
@@ -159,9 +165,20 @@ class TestRubyLexer < Minitest::Test
159
165
  assert_nil @lex.next_token
160
166
  end
161
167
 
162
- def test_unicode_ident
163
- s = "@\u1088\u1077\u1093\u1072"
164
- assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
168
+ def test_pct_w_backslashes
169
+ ["\t", "\n", "\r", "\v", "\f"].each do |char|
170
+ next if !RubyLexer::HAS_ENC and char == "\v"
171
+
172
+ assert_lex("%w[foo#{char}bar]",
173
+ s(:array, s(:str, "foo"), s(:str, "bar")),
174
+
175
+ :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
176
+ :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
177
+ :tSPACE, nil, EXPR_BEG, 0, 0,
178
+ :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
179
+ :tSPACE, nil, EXPR_BEG, 0, 0,
180
+ :tSTRING_END, nil, EXPR_LIT, 0, 0)
181
+ end
165
182
  end
166
183
 
167
184
  def test_read_escape
@@ -217,6 +234,90 @@ class TestRubyLexer < Minitest::Test
217
234
  assert_read_escape "\230", 'M-\cx'
218
235
  end
219
236
 
237
+ def test_ruby21_imaginary_literal
238
+ setup_lexer_class RubyParser::V21
239
+
240
+ assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
241
+ assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
242
+ assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
243
+ assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
244
+ assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
245
+ assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
246
+ assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
247
+ assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
248
+ assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
249
+ assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
250
+
251
+ c010 = Complex(0, 10)
252
+ assert_lex3("-10i", nil,
253
+ :tUMINUS_NUM, "-", EXPR_BEG,
254
+ :tIMAGINARY, c010, EXPR_NUM)
255
+ end
256
+
257
+ def test_ruby21_imaginary_literal_with_succeeding_keyword
258
+ setup_lexer_class RubyParser::V21
259
+
260
+ # 2/4 scenarios are syntax errors on all tested versions so I
261
+ # deleted them.
262
+
263
+ assert_lex3("1if", nil,
264
+ :tINTEGER, 1, EXPR_NUM,
265
+ :kIF_MOD, "if", EXPR_PAR)
266
+ assert_lex3("1.0if", nil,
267
+ :tFLOAT, 1.0, EXPR_NUM,
268
+ :kIF_MOD, "if", EXPR_PAR)
269
+ end
270
+
271
+ def test_ruby21_rational_imaginary_literal
272
+ setup_lexer_class RubyParser::V21
273
+
274
+ assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
275
+ assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
276
+ assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
277
+ assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
278
+ assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
279
+ assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
280
+ assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
281
+ assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
282
+ assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
283
+ assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
284
+
285
+ assert_lex3("-10ri", nil,
286
+ :tUMINUS_NUM, "-", EXPR_BEG,
287
+ :tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
288
+ end
289
+
290
+ def test_ruby21_rational_literal
291
+ setup_lexer_class RubyParser::V21
292
+
293
+ assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
294
+ assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
295
+ assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
296
+ assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
297
+ assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
298
+ assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
299
+ assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
300
+ assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
301
+ assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
302
+ assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
303
+
304
+ r10 = Rational(10)
305
+ assert_lex3("-10r", nil,
306
+ :tUMINUS_NUM, "-", EXPR_BEG,
307
+ :tRATIONAL, r10, EXPR_NUM)
308
+ end
309
+
310
+ def test_unicode_ident
311
+ s = "@\u1088\u1077\u1093\u1072"
312
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
313
+ end
314
+
315
+ def test_why_does_ruby_hate_me?
316
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
317
+ nil,
318
+ :tSTRING, %W[ Nl% \u0000 \u0000 A \u0000 999 ].join, EXPR_END)
319
+ end
320
+
220
321
  def test_yylex_ambiguous_uminus
221
322
  assert_lex3("m -3",
222
323
  nil,
@@ -248,6 +349,15 @@ class TestRubyLexer < Minitest::Test
248
349
  assert_lex3("&&=", nil, :tOP_ASGN, "&&", EXPR_BEG)
249
350
  end
250
351
 
352
+ def test_yylex_and_arg
353
+ self.lex_state = EXPR_ARG
354
+
355
+ assert_lex3(" &y",
356
+ nil,
357
+ :tAMPER, "&", EXPR_BEG,
358
+ :tIDENTIFIER, "y", EXPR_ARG)
359
+ end
360
+
251
361
  def test_yylex_and_dot
252
362
  setup_lexer_class RubyParser::V23
253
363
 
@@ -272,15 +382,6 @@ class TestRubyLexer < Minitest::Test
272
382
  :tIDENTIFIER, "y")
273
383
  end
274
384
 
275
- def test_yylex_and_arg
276
- self.lex_state = EXPR_ARG
277
-
278
- assert_lex3(" &y",
279
- nil,
280
- :tAMPER, "&", EXPR_BEG,
281
- :tIDENTIFIER, "y", EXPR_ARG)
282
- end
283
-
284
385
  def test_yylex_and_equals
285
386
  assert_lex3("&=", nil, :tOP_ASGN, "&", EXPR_BEG)
286
387
  end
@@ -303,424 +404,112 @@ class TestRubyLexer < Minitest::Test
303
404
  assert_lex3 "=>", nil, :tASSOC, "=>", EXPR_BEG
304
405
  end
305
406
 
306
- def test_yylex_label__20
307
- setup_lexer_class RubyParser::V20
407
+ def test_yylex_back_ref
408
+ assert_lex3("[$&, $`, $', $+]",
409
+ nil,
410
+ :tLBRACK, "[", EXPR_PAR,
411
+ :tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
412
+ :tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
413
+ :tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
414
+ :tBACK_REF, :+, EXPR_END,
415
+ :tRBRACK, "]", EXPR_END)
416
+ end
308
417
 
309
- assert_lex3("{a:",
418
+ def test_yylex_backslash
419
+ assert_lex3("1 \\\n+ 2",
310
420
  nil,
311
- :tLBRACE, "{", EXPR_PAR,
312
- :tLABEL, "a", EXPR_LAB)
421
+ :tINTEGER, 1, EXPR_NUM,
422
+ :tPLUS, "+", EXPR_BEG,
423
+ :tINTEGER, 2, EXPR_NUM)
313
424
  end
314
425
 
315
- def test_yylex_label_in_params__20
316
- setup_lexer_class RubyParser::V20
426
+ def test_yylex_backslash_bad
427
+ refute_lex("1 \\ + 2", :tINTEGER, 1)
428
+ end
317
429
 
318
- assert_lex3("foo(a:",
430
+ def test_yylex_backtick
431
+ assert_lex3("`ls`",
319
432
  nil,
320
- :tIDENTIFIER, "foo", EXPR_CMDARG,
321
- :tLPAREN2, "(", EXPR_PAR,
322
- :tLABEL, "a", EXPR_LAB)
433
+ :tXSTRING_BEG, "`", EXPR_BEG,
434
+ :tSTRING_CONTENT, "ls", EXPR_BEG,
435
+ :tSTRING_END, "`", EXPR_LIT)
323
436
  end
324
437
 
325
- def test_yylex_paren_string_parens_interpolated
326
- setup_lexer('%((#{b}#{d}))',
327
- s(:dstr,
328
- "(",
329
- s(:evstr, s(:call, nil, :b)),
330
- s(:evstr, s(:call, nil, :d)),
331
- s(:str, ")")))
438
+ def test_yylex_backtick_cmdarg
439
+ self.lex_state = EXPR_DOT
332
440
 
333
- assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
334
- assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
335
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
441
+ # \n ensures expr_cmd (TODO: why?)
442
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
443
+ end
336
444
 
337
- emulate_string_interpolation do
338
- assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
339
- end
445
+ def test_yylex_backtick_dot
446
+ self.lex_state = EXPR_DOT
340
447
 
341
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
448
+ assert_lex3("a.`(3)",
449
+ nil,
450
+ :tIDENTIFIER, "a", EXPR_CMDARG,
451
+ :tDOT, ".", EXPR_DOT,
452
+ :tBACK_REF2, "`", EXPR_ARG,
453
+ :tLPAREN2, "(", EXPR_PAR,
454
+ :tINTEGER, 3, EXPR_NUM,
455
+ :tRPAREN, ")", EXPR_ENDFN)
456
+ end
342
457
 
343
- emulate_string_interpolation do
344
- assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
345
- end
458
+ def test_yylex_backtick_method
459
+ self.lex_state = EXPR_FNAME
346
460
 
347
- assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
348
- assert_next_lexeme :tSTRING_END, ")", EXPR_END, 0, 0
461
+ assert_lex3("`",
462
+ nil,
463
+ :tBACK_REF2, "`", EXPR_END)
464
+ end
349
465
 
350
- refute_lexeme
466
+ def test_yylex_bad_char
467
+ refute_lex(" \010 ")
351
468
  end
352
469
 
353
- def test_yylex_paren_string_interpolated_regexp
354
- setup_lexer('%( #{(/abcd/)} )',
355
- s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
470
+ def test_yylex_bang
471
+ assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
472
+ end
356
473
 
357
- assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
358
- assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
359
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
474
+ def test_yylex_bang_equals
475
+ assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
476
+ end
360
477
 
361
- emulate_string_interpolation do
362
- assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
363
- assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
364
- assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
365
- assert_next_lexeme :tREGEXP_END, "", EXPR_END, 1, 0
366
- assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
367
- end
478
+ def test_yylex_bang_tilde
479
+ assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
480
+ end
368
481
 
369
- assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
370
- assert_next_lexeme :tSTRING_END, ")", EXPR_END, 0, 0
482
+ def test_yylex_block_bug_1
483
+ assert_lex3("a do end",
484
+ s(:iter, s(:call, nil, :a), 0),
371
485
 
372
- refute_lexeme
486
+ :tIDENTIFIER, "a", EXPR_CMDARG,
487
+ :kDO, "do", EXPR_BEG,
488
+ :kEND, "end", EXPR_END)
373
489
  end
374
490
 
375
- def test_yylex_not_at_defn__20
376
- setup_lexer_class RubyParser::V20
491
+ def test_yylex_block_bug_2
492
+ assert_lex3("a = 1\na do\nend",
493
+ s(:block,
494
+ s(:lasgn, :a, s(:lit, 1)),
495
+ s(:iter, s(:call, nil, :a), 0)),
377
496
 
378
- assert_lex("def +@; end",
379
- s(:defn, :+@, s(:args), s(:nil)),
380
-
381
- :kDEF, "def", EXPR_FNAME, 0, 0,
382
- :tUPLUS, "+@", EXPR_ARG, 0, 0,
383
- :tSEMI, ";", EXPR_BEG, 0, 0,
384
- :kEND, "end", EXPR_END, 0, 0)
385
-
386
- assert_lex("def !@; end",
387
- s(:defn, :"!@", s(:args), s(:nil)),
388
-
389
- :kDEF, "def", EXPR_FNAME, 0, 0,
390
- :tUBANG, "!@", EXPR_ARG, 0, 0,
391
- :tSEMI, ";", EXPR_BEG, 0, 0,
392
- :kEND, "end", EXPR_END, 0, 0)
393
- end
394
-
395
- def test_yylex_not_at_ivar
396
- assert_lex("!@ivar",
397
- s(:call, s(:ivar, :@ivar), :"!"),
398
-
399
- :tBANG, "!", EXPR_BEG, 0, 0,
400
- :tIVAR, "@ivar", EXPR_END, 0, 0)
401
- end
402
-
403
- def test_yylex_number_times_ident_times_return_number
404
- assert_lex("1 * b * 3",
405
- s(:call,
406
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
407
- :*, s(:lit, 3)),
408
-
409
- :tINTEGER, 1, EXPR_NUM, 0, 0,
410
- :tSTAR2, "*", EXPR_BEG, 0, 0,
411
- :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
412
- :tSTAR2, "*", EXPR_BEG, 0, 0,
413
- :tINTEGER, 3, EXPR_NUM, 0, 0)
414
-
415
- assert_lex("1 * b *\n 3",
416
- s(:call,
417
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
418
- :*, s(:lit, 3)),
419
-
420
- :tINTEGER, 1, EXPR_NUM, 0, 0,
421
- :tSTAR2, "*", EXPR_BEG, 0, 0,
422
- :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
423
- :tSTAR2, "*", EXPR_BEG, 0, 0,
424
- :tINTEGER, 3, EXPR_NUM, 0, 0)
425
- end
426
-
427
- def test_yylex_paren_string_parens_interpolated_regexp
428
- setup_lexer('%((#{(/abcd/)}))',
429
- s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
430
-
431
- assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
432
- assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
433
-
434
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
435
-
436
- emulate_string_interpolation do
437
- assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
438
- assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
439
- assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
440
- assert_next_lexeme :tREGEXP_END, "", EXPR_END, 1, 0
441
- assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
442
- end
443
-
444
- assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
445
- assert_next_lexeme :tSTRING_END, ")", EXPR_END, 0, 0
446
-
447
- refute_lexeme
448
- end
449
-
450
- def test_yylex_method_parens_chevron
451
- assert_lex("a()<<1",
452
- s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
453
- :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
454
- :tLPAREN2, "(", EXPR_PAR, 1, 0,
455
- :tRPAREN, ")", EXPR_ENDFN, 0, 0,
456
- :tLSHFT, "<<" , EXPR_BEG, 0, 0,
457
- :tINTEGER, 1, EXPR_NUM, 0, 0)
458
- end
459
-
460
- def test_yylex_lambda_args__20
461
- setup_lexer_class RubyParser::V20
462
-
463
- assert_lex("-> (a) { }",
464
- s(:iter, s(:call, nil, :lambda),
465
- s(:args, :a)),
466
-
467
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
468
- :tLPAREN2, "(", EXPR_PAR, 1, 0,
469
- :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
470
- :tRPAREN, ")", EXPR_ENDFN, 0, 0,
471
- :tLCURLY, "{", EXPR_PAR, 0, 1,
472
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
473
- end
474
-
475
- def test_yylex_lambda_as_args_with_block__20
476
- setup_lexer_class RubyParser::V20
477
-
478
- assert_lex3("a -> do end do end",
479
- nil,
480
- :tIDENTIFIER, "a", EXPR_CMDARG,
481
- :tLAMBDA, nil, EXPR_ENDFN,
482
- :kDO, "do", EXPR_BEG,
483
- :kEND, "end", EXPR_END,
484
- :kDO, "do", EXPR_BEG,
485
- :kEND, "end", EXPR_END)
486
- end
487
-
488
- def test_yylex_lambda_args_opt__20
489
- setup_lexer_class RubyParser::V20
490
-
491
- assert_lex("-> (a=nil) { }",
492
- s(:iter, s(:call, nil, :lambda),
493
- s(:args, s(:lasgn, :a, s(:nil)))),
494
-
495
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
496
- :tLPAREN2, "(", EXPR_PAR, 1, 0,
497
- :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
498
- :tEQL, "=", EXPR_BEG, 1, 0,
499
- :kNIL, "nil", EXPR_END, 1, 0,
500
- :tRPAREN, ")", EXPR_ENDFN, 0, 0,
501
- :tLCURLY, "{", EXPR_PAR, 0, 1,
502
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
503
- end
504
-
505
- def test_yylex_lambda_hash__20
506
- setup_lexer_class RubyParser::V20
507
-
508
- assert_lex("-> (a={}) { }",
509
- s(:iter, s(:call, nil, :lambda),
510
- s(:args, s(:lasgn, :a, s(:hash)))),
511
-
512
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
513
- :tLPAREN2, "(", EXPR_PAR, 1, 0,
514
- :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
515
- :tEQL, "=", EXPR_BEG, 1, 0,
516
- :tLBRACE, "{", EXPR_PAR, 1, 1,
517
- :tRCURLY, "}", EXPR_ENDARG, 1, 0,
518
- :tRPAREN, ")", EXPR_ENDFN, 0, 0,
519
- :tLCURLY, "{", EXPR_PAR, 0, 1,
520
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
521
- end
522
-
523
- def test_yylex_iter_array_curly
524
- assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
525
- s(:iter,
526
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
527
- s(:args, :c, :d)),
528
-
529
- :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
530
- :tSYMBOL, "a", EXPR_END, 0, 0,
531
- :tCOMMA, ",", EXPR_PAR, 0, 0,
532
- :tLBRACK, "[", EXPR_PAR, 1, 0,
533
- :tSYMBOL, "b", EXPR_END, 1, 0,
534
- :tRBRACK, "]", EXPR_ENDARG, 0, 0,
535
- :tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
536
- :tPIPE, "|", EXPR_PAR, 0, 1,
537
- :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
538
- :tCOMMA, ",", EXPR_PAR, 0, 1,
539
- :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
540
- :tPIPE, "|", EXPR_PAR, 0, 1,
541
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
542
- end
543
-
544
- def test_yylex_const_call_same_name
545
- assert_lex("X = a { }; b { f :c }",
546
- s(:block,
547
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
548
- s(:iter,
549
- s(:call, nil, :b),
550
- 0,
551
- s(:call, nil, :f, s(:lit, :c)))),
552
-
553
- :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
554
- :tEQL, "=", EXPR_BEG, 0, 0,
555
- :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
556
- :tLCURLY, "{", EXPR_PAR, 0, 1,
557
- :tRCURLY, "}", EXPR_ENDARG, 0, 0,
558
- :tSEMI, ";", EXPR_BEG, 0, 0,
559
-
560
- :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
561
- :tLCURLY, "{", EXPR_PAR, 0, 1,
562
- :tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
563
- :tSYMBOL, "c", EXPR_END, 0, 1,
564
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
565
-
566
- assert_lex("X = a { }; b { X :c }",
567
- s(:block,
568
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
569
- s(:iter,
570
- s(:call, nil, :b),
571
- 0,
572
- s(:call, nil, :X, s(:lit, :c)))),
573
-
574
- :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
575
- :tEQL, "=", EXPR_BEG, 0, 0,
576
- :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
577
- :tLCURLY, "{", EXPR_PAR, 0, 1,
578
- :tRCURLY, "}", EXPR_ENDARG, 0, 0,
579
- :tSEMI, ";", EXPR_BEG, 0, 0,
580
-
581
- :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
582
- :tLCURLY, "{", EXPR_PAR, 0, 1,
583
- :tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
584
- :tSYMBOL, "c", EXPR_END, 0, 1,
585
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
586
- end
587
-
588
- def test_yylex_lasgn_call_same_name
589
- assert_lex("a = b.c :d => 1",
590
- s(:lasgn, :a,
591
- s(:call, s(:call, nil, :b), :c,
592
- s(:hash, s(:lit, :d), s(:lit, 1)))),
593
-
594
- :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
595
- :tEQL, "=", EXPR_BEG, 0, 0,
596
- :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
597
- :tDOT, ".", EXPR_DOT, 0, 0,
598
- :tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
599
- :tSYMBOL, "d", EXPR_END, 0, 0,
600
- :tASSOC, "=>", EXPR_BEG, 0, 0,
601
- :tINTEGER, 1, EXPR_NUM, 0, 0)
602
-
603
- assert_lex("a = b.a :d => 1",
604
- s(:lasgn, :a,
605
- s(:call, s(:call, nil, :b), :a,
606
- s(:hash, s(:lit, :d), s(:lit, 1)))),
607
-
608
- :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
609
- :tEQL, "=", EXPR_BEG, 0, 0,
610
- :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
611
- :tDOT, ".", EXPR_DOT, 0, 0,
612
- :tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
613
- :tSYMBOL, "d", EXPR_END, 0, 0,
614
- :tASSOC, "=>", EXPR_BEG, 0, 0,
615
- :tINTEGER, 1, EXPR_NUM, 0, 0)
616
- end
617
-
618
- def test_yylex_back_ref
619
- assert_lex3("[$&, $`, $', $+]",
620
- nil,
621
- :tLBRACK, "[", EXPR_PAR,
622
- :tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
623
- :tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
624
- :tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
625
- :tBACK_REF, :+, EXPR_END,
626
- :tRBRACK, "]", EXPR_ENDARG)
627
- end
628
-
629
- def test_yylex_backslash
630
- assert_lex3("1 \\\n+ 2",
631
- nil,
632
- :tINTEGER, 1, EXPR_NUM,
633
- :tPLUS, "+", EXPR_BEG,
634
- :tINTEGER, 2, EXPR_NUM)
635
- end
636
-
637
- def test_yylex_backslash_bad
638
- refute_lex("1 \\ + 2", :tINTEGER, 1)
639
- end
640
-
641
- def test_yylex_backtick
642
- assert_lex3("`ls`",
643
- nil,
644
- :tXSTRING_BEG, "`", EXPR_BEG,
645
- :tSTRING_CONTENT, "ls", EXPR_BEG,
646
- :tSTRING_END, "`", EXPR_END)
647
- end
648
-
649
- def test_yylex_backtick_cmdarg
650
- self.lex_state = EXPR_DOT
651
-
652
- # \n ensures expr_cmd (TODO: why?)
653
- assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
654
- end
655
-
656
- def test_yylex_backtick_dot
657
- self.lex_state = EXPR_DOT
658
-
659
- assert_lex3("a.`(3)",
660
- nil,
661
- :tIDENTIFIER, "a", EXPR_CMDARG,
662
- :tDOT, ".", EXPR_DOT,
663
- :tBACK_REF2, "`", EXPR_ARG,
664
- :tLPAREN2, "(", EXPR_PAR,
665
- :tINTEGER, 3, EXPR_NUM,
666
- :tRPAREN, ")", EXPR_ENDFN)
667
- end
668
-
669
- def test_yylex_backtick_method
670
- self.lex_state = EXPR_FNAME
671
-
672
- assert_lex3("`",
673
- nil,
674
- :tBACK_REF2, "`", EXPR_END)
675
- end
676
-
677
- def test_yylex_bad_char
678
- refute_lex(" \010 ")
679
- end
680
-
681
- def test_yylex_bang
682
- assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
683
- end
684
-
685
- def test_yylex_bang_equals
686
- assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
687
- end
688
-
689
- def test_yylex_bang_tilde
690
- assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
691
- end
692
-
693
- def test_yylex_block_bug_1
694
- assert_lex3("a do end",
695
- s(:iter, s(:call, nil, :a), 0),
696
-
697
- :tIDENTIFIER, "a", EXPR_CMDARG,
698
- :kDO, "do", EXPR_BEG,
699
- :kEND, "end", EXPR_END)
700
- end
701
-
702
- def test_yylex_block_bug_2
703
- assert_lex3("a = 1\na do\nend",
704
- s(:block,
705
- s(:lasgn, :a, s(:lit, 1)),
706
- s(:iter, s(:call, nil, :a), 0)),
707
-
708
- :tIDENTIFIER, "a", EXPR_CMDARG,
709
- :tEQL, "=", EXPR_BEG,
710
- :tINTEGER, 1, EXPR_NUM,
711
- :tNL, nil, EXPR_BEG,
712
- :tIDENTIFIER, "a", EXPR_CMDARG,
713
- :kDO, "do", EXPR_BEG,
714
- :kEND, "end", EXPR_END)
715
- end
497
+ :tIDENTIFIER, "a", EXPR_CMDARG,
498
+ :tEQL, "=", EXPR_BEG,
499
+ :tINTEGER, 1, EXPR_NUM,
500
+ :tNL, nil, EXPR_BEG,
501
+ :tIDENTIFIER, "a", EXPR_CMDARG,
502
+ :kDO, "do", EXPR_BEG,
503
+ :kEND, "end", EXPR_END)
504
+ end
716
505
 
717
506
  def test_yylex_block_bug_3
718
507
  assert_lex3("a { }",
719
508
  s(:iter, s(:call, nil, :a), 0),
720
509
 
721
510
  :tIDENTIFIER, "a", EXPR_CMDARG, # verified
722
- :tLCURLY, "{", EXPR_PAR,
723
- :tRCURLY, "}", EXPR_ENDARG)
511
+ :tLCURLY, "{", EXPR_PAR,
512
+ :tRCURLY, "}", EXPR_END)
724
513
  end
725
514
 
726
515
  def test_yylex_carat
@@ -766,9 +555,9 @@ class TestRubyLexer < Minitest::Test
766
555
  def test_yylex_comment
767
556
  assert_lex3("1 # one\n# two\n2",
768
557
  nil,
769
- :tINTEGER, 1, EXPR_NUM,
558
+ :tINTEGER, 1, EXPR_NUM,
770
559
  :tNL, nil, EXPR_BEG,
771
- :tINTEGER, 2, EXPR_NUM)
560
+ :tINTEGER, 2, EXPR_NUM)
772
561
 
773
562
  assert_equal "# one\n# two\n", @lex.comments
774
563
  end
@@ -776,7 +565,7 @@ class TestRubyLexer < Minitest::Test
776
565
  def test_yylex_comment_begin
777
566
  assert_lex3("=begin\nblah\nblah\n=end\n42",
778
567
  nil,
779
- :tINTEGER, 42, EXPR_NUM)
568
+ :tINTEGER, 42, EXPR_NUM)
780
569
 
781
570
  assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
782
571
  end
@@ -792,7 +581,7 @@ class TestRubyLexer < Minitest::Test
792
581
  nil,
793
582
  :tIDENTIFIER, "beginfoo", EXPR_CMDARG,
794
583
  :tEQL, "=", EXPR_BEG,
795
- :tINTEGER, 5, EXPR_NUM,
584
+ :tINTEGER, 5, EXPR_NUM,
796
585
  :tNL, nil, EXPR_BEG,
797
586
  :tIDENTIFIER, "p", EXPR_CMDARG,
798
587
  :tIDENTIFIER, "x", EXPR_ARG,
@@ -816,6 +605,50 @@ class TestRubyLexer < Minitest::Test
816
605
  assert_lex3("# comment", nil)
817
606
  end
818
607
 
608
+ def test_yylex_const_call_same_name
609
+ assert_lex("X = a { }; b { f :c }",
610
+ s(:block,
611
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
612
+ s(:iter,
613
+ s(:call, nil, :b),
614
+ 0,
615
+ s(:call, nil, :f, s(:lit, :c)))),
616
+
617
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
618
+ :tEQL, "=", EXPR_BEG, 0, 0,
619
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
620
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
621
+ :tRCURLY, "}", EXPR_END, 0, 0,
622
+ :tSEMI, ";", EXPR_BEG, 0, 0,
623
+
624
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
625
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
626
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
627
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
628
+ :tRCURLY, "}", EXPR_END, 0, 0)
629
+
630
+ assert_lex("X = a { }; b { X :c }",
631
+ s(:block,
632
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
633
+ s(:iter,
634
+ s(:call, nil, :b),
635
+ 0,
636
+ s(:call, nil, :X, s(:lit, :c)))),
637
+
638
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
639
+ :tEQL, "=", EXPR_BEG, 0, 0,
640
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
641
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
642
+ :tRCURLY, "}", EXPR_END, 0, 0,
643
+ :tSEMI, ";", EXPR_BEG, 0, 0,
644
+
645
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
646
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
647
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
648
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
649
+ :tRCURLY, "}", EXPR_END, 0, 0)
650
+ end
651
+
819
652
  def test_yylex_constant
820
653
  assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", EXPR_CMDARG)
821
654
  end
@@ -847,7 +680,7 @@ class TestRubyLexer < Minitest::Test
847
680
  nil,
848
681
  :tIDENTIFIER, "a", EXPR_CMDARG,
849
682
  :tDIVIDE, "/", EXPR_BEG,
850
- :tINTEGER, 2, EXPR_NUM)
683
+ :tINTEGER, 2, EXPR_NUM)
851
684
  end
852
685
 
853
686
  def test_yylex_div_equals
@@ -855,7 +688,7 @@ class TestRubyLexer < Minitest::Test
855
688
  nil,
856
689
  :tIDENTIFIER, "a", EXPR_CMDARG,
857
690
  :tOP_ASGN, "/", EXPR_BEG,
858
- :tINTEGER, 2, EXPR_NUM)
691
+ :tINTEGER, 2, EXPR_NUM)
859
692
  end
860
693
 
861
694
  def test_yylex_do
@@ -863,7 +696,7 @@ class TestRubyLexer < Minitest::Test
863
696
  nil,
864
697
  :tIDENTIFIER, "x", EXPR_CMDARG,
865
698
  :kDO, "do", EXPR_BEG,
866
- :tINTEGER, 42, EXPR_NUM,
699
+ :tINTEGER, 42, EXPR_NUM,
867
700
  :kEND, "end", EXPR_END)
868
701
  end
869
702
 
@@ -876,7 +709,7 @@ class TestRubyLexer < Minitest::Test
876
709
  :tDOT, ".", EXPR_DOT,
877
710
  :tIDENTIFIER, "y", EXPR_ARG,
878
711
  :kDO_BLOCK, "do", EXPR_BEG,
879
- :tINTEGER, 42, EXPR_NUM,
712
+ :tINTEGER, 42, EXPR_NUM,
880
713
  :kEND, "end", EXPR_END) do
881
714
  @lex.cmdarg.push true
882
715
  end
@@ -888,24 +721,16 @@ class TestRubyLexer < Minitest::Test
888
721
  assert_lex3("do 42 end",
889
722
  nil,
890
723
  :kDO_BLOCK, "do", EXPR_BEG,
891
- :tINTEGER, 42, EXPR_NUM,
724
+ :tINTEGER, 42, EXPR_NUM,
892
725
  :kEND, "end", EXPR_END)
893
726
  end
894
727
 
895
- def test_yylex_is_your_spacebar_broken?
896
- assert_lex3(":a!=:b",
897
- nil,
898
- :tSYMBOL, "a", EXPR_END,
899
- :tNEQ, "!=", EXPR_BEG,
900
- :tSYMBOL, "b", EXPR_END)
901
- end
902
-
903
728
  def test_yylex_do_cond
904
729
  assert_lex3("x do 42 end",
905
730
  nil,
906
731
  :tIDENTIFIER, "x", EXPR_CMDARG,
907
732
  :kDO_COND, "do", EXPR_BEG,
908
- :tINTEGER, 42, EXPR_NUM,
733
+ :tINTEGER, 42, EXPR_NUM,
909
734
  :kEND, "end", EXPR_END) do
910
735
  @lex.cond.push true
911
736
  end
@@ -950,7 +775,7 @@ class TestRubyLexer < Minitest::Test
950
775
  end
951
776
 
952
777
  def test_yylex_float
953
- assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
778
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
954
779
  end
955
780
 
956
781
  def test_yylex_float_bad_no_underscores
@@ -968,7 +793,7 @@ class TestRubyLexer < Minitest::Test
968
793
  def test_yylex_float_call
969
794
  assert_lex3("1.0.to_s",
970
795
  nil,
971
- :tFLOAT, 1.0, EXPR_NUM,
796
+ :tFLOAT, 1.0, EXPR_NUM,
972
797
  :tDOT, ".", EXPR_DOT,
973
798
  :tIDENTIFIER, "to_s", EXPR_ARG)
974
799
  end
@@ -983,7 +808,7 @@ class TestRubyLexer < Minitest::Test
983
808
  assert_lex3("-1.0E10",
984
809
  nil,
985
810
  :tUMINUS_NUM, "-", EXPR_BEG,
986
- :tFLOAT, 10000000000.0, EXPR_NUM)
811
+ :tFLOAT, 10000000000.0, EXPR_NUM)
987
812
  end
988
813
 
989
814
  def test_yylex_float_dot_e
@@ -996,29 +821,22 @@ class TestRubyLexer < Minitest::Test
996
821
  assert_lex3("-1.0e10",
997
822
  nil,
998
823
  :tUMINUS_NUM, "-", EXPR_BEG,
999
- :tFLOAT, 10000000000.0, EXPR_NUM)
824
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1000
825
  end
1001
826
 
1002
827
  def test_yylex_float_e
1003
828
  assert_lex3("1e10",
1004
829
  nil,
1005
- :tFLOAT, 10000000000.0, EXPR_NUM)
830
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1006
831
  end
1007
832
 
1008
833
  def test_yylex_float_e_bad_double_e
1009
834
  assert_lex3("1e2e3",
1010
835
  nil,
1011
- :tFLOAT, 100, EXPR_NUM,
836
+ :tFLOAT, 100, EXPR_NUM,
1012
837
  :tIDENTIFIER, "e3", EXPR_END)
1013
838
  end
1014
839
 
1015
- def test_yylex_float_if_modifier
1016
- assert_lex3("1e2if",
1017
- nil,
1018
- :tFLOAT, 100, EXPR_NUM,
1019
- :kIF_MOD, "if", EXPR_PAR)
1020
- end
1021
-
1022
840
  def test_yylex_float_e_bad_trailing_underscore
1023
841
  refute_lex "123_e10"
1024
842
  end
@@ -1031,21 +849,21 @@ class TestRubyLexer < Minitest::Test
1031
849
  assert_lex3("-1e10",
1032
850
  nil,
1033
851
  :tUMINUS_NUM, "-", EXPR_BEG,
1034
- :tFLOAT, 10000000000.0, EXPR_NUM)
852
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1035
853
  end
1036
854
 
1037
855
  def test_yylex_float_e_neg_minus
1038
856
  assert_lex3("-1e-10",
1039
857
  nil,
1040
858
  :tUMINUS_NUM, "-", EXPR_BEG,
1041
- :tFLOAT, 1.0e-10, EXPR_NUM)
859
+ :tFLOAT, 1.0e-10, EXPR_NUM)
1042
860
  end
1043
861
 
1044
862
  def test_yylex_float_e_neg_plus
1045
863
  assert_lex3("-1e+10",
1046
864
  nil,
1047
865
  :tUMINUS_NUM, "-", EXPR_BEG,
1048
- :tFLOAT, 10000000000.0, EXPR_NUM)
866
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1049
867
  end
1050
868
 
1051
869
  def test_yylex_float_e_plus
@@ -1056,11 +874,18 @@ class TestRubyLexer < Minitest::Test
1056
874
  assert_lex3("0e0", nil, :tFLOAT, 0.0, EXPR_NUM)
1057
875
  end
1058
876
 
877
+ def test_yylex_float_if_modifier
878
+ assert_lex3("1e2if",
879
+ nil,
880
+ :tFLOAT, 100, EXPR_NUM,
881
+ :kIF_MOD, "if", EXPR_PAR)
882
+ end
883
+
1059
884
  def test_yylex_float_neg
1060
885
  assert_lex3("-1.0",
1061
886
  nil,
1062
887
  :tUMINUS_NUM, "-", EXPR_BEG,
1063
- :tFLOAT, 1.0, EXPR_NUM)
888
+ :tFLOAT, 1.0, EXPR_NUM)
1064
889
  end
1065
890
 
1066
891
  def test_yylex_ge
@@ -1068,7 +893,7 @@ class TestRubyLexer < Minitest::Test
1068
893
  nil,
1069
894
  :tIDENTIFIER, "a", EXPR_CMDARG,
1070
895
  :tGEQ, ">=", EXPR_BEG,
1071
- :tINTEGER, 2, EXPR_NUM)
896
+ :tINTEGER, 2, EXPR_NUM)
1072
897
  end
1073
898
 
1074
899
  def test_yylex_global
@@ -1121,27 +946,113 @@ class TestRubyLexer < Minitest::Test
1121
946
  :tGVAR, "$<", EXPR_END, :tCOMMA, ",", EXPR_PAR,
1122
947
  :tGVAR, "$>", EXPR_END, :tCOMMA, ",", EXPR_PAR,
1123
948
  :tGVAR, "$\"", EXPR_END,
1124
- :tRBRACK, "]", EXPR_ENDARG)
949
+ :tRBRACK, "]", EXPR_END)
950
+ end
951
+
952
+ def test_yylex_global_underscore
953
+ assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
954
+ end
955
+
956
+ def test_yylex_global_wierd
957
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
958
+ end
959
+
960
+ def test_yylex_global_zero
961
+ assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
962
+ end
963
+
964
+ def test_yylex_gt
965
+ assert_lex3("a > 2",
966
+ nil,
967
+ :tIDENTIFIER, "a", EXPR_CMDARG,
968
+ :tGT, ">", EXPR_BEG,
969
+ :tINTEGER, 2, EXPR_NUM)
970
+ end
971
+
972
+ def test_yylex_hash_colon
973
+ assert_lex("{a:1}",
974
+ s(:hash, s(:lit, :a), s(:lit, 1)),
975
+
976
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
977
+ :tLABEL, "a", EXPR_LAB, 0, 1,
978
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
979
+ :tRCURLY, "}", EXPR_END, 0, 0)
980
+ end
981
+
982
+ def test_yylex_hash_colon_double_quoted_symbol
983
+ assert_lex('{"abc": :b}',
984
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
985
+
986
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
987
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
988
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
989
+ :tRCURLY, "}", EXPR_END, 0, 0)
990
+ end
991
+
992
+ def test_yylex_hash_colon_double_quoted_symbol_22
993
+ setup_lexer_class RubyParser::V22
994
+
995
+ assert_lex('{"abc": :b}',
996
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
997
+
998
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
999
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1000
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1001
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1125
1002
  end
1126
1003
 
1127
- def test_yylex_global_underscore
1128
- assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
1004
+ def test_yylex_hash_colon_double_quoted_with_escapes
1005
+ assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
1006
+ nil,
1007
+
1008
+ :tLBRACE, "{", EXPR_PAR,
1009
+ :tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
1010
+ :tINTEGER, 1, EXPR_NUM,
1011
+ :tRCURLY, "}", EXPR_END)
1129
1012
  end
1130
1013
 
1131
- def test_yylex_global_wierd
1132
- assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
1014
+ def test_yylex_hash_colon_quoted_22
1015
+ setup_lexer_class RubyParser::V22
1016
+
1017
+ assert_lex("{'a':1}",
1018
+ s(:hash, s(:lit, :a), s(:lit, 1)),
1019
+
1020
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1021
+ :tLABEL, "a", EXPR_LAB, 0, 1,
1022
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
1023
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1133
1024
  end
1134
1025
 
1135
- def test_yylex_global_zero
1136
- assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
1026
+ def test_yylex_hash_colon_quoted_symbol
1027
+ assert_lex("{'abc': :b}",
1028
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1029
+
1030
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1031
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1032
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1033
+ :tRCURLY, "}", EXPR_END, 0, 0)
1137
1034
  end
1138
1035
 
1139
- def test_yylex_gt
1140
- assert_lex3("a > 2",
1141
- nil,
1142
- :tIDENTIFIER, "a", EXPR_CMDARG,
1143
- :tGT, ">", EXPR_BEG,
1144
- :tINTEGER, 2, EXPR_NUM)
1036
+ def test_yylex_hash_colon_quoted_symbol_22
1037
+ setup_lexer_class RubyParser::V22
1038
+
1039
+ assert_lex("{'abc': :b}",
1040
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1041
+
1042
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1043
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1044
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1045
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1046
+ end
1047
+
1048
+ def test_yylex_hash_colon_quoted_with_escapes
1049
+ assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
1050
+ nil,
1051
+
1052
+ :tLBRACE, "{", EXPR_PAR,
1053
+ :tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
1054
+ :tINTEGER, 1, EXPR_NUM,
1055
+ :tRCURLY, "}", EXPR_END)
1145
1056
  end
1146
1057
 
1147
1058
  def test_yylex_heredoc_backtick
@@ -1151,7 +1062,7 @@ class TestRubyLexer < Minitest::Test
1151
1062
  :tEQL, "=", EXPR_BEG,
1152
1063
  :tXSTRING_BEG, "`", EXPR_BEG,
1153
1064
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1154
- :tSTRING_END, "EOF", EXPR_END,
1065
+ :tSTRING_END, "EOF", EXPR_LIT,
1155
1066
  :tNL, nil, EXPR_BEG)
1156
1067
  end
1157
1068
 
@@ -1162,7 +1073,7 @@ class TestRubyLexer < Minitest::Test
1162
1073
  :tEQL, "=", EXPR_BEG,
1163
1074
  :tSTRING_BEG, "\"", EXPR_BEG,
1164
1075
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1165
- :tSTRING_END, "EOF", EXPR_END,
1076
+ :tSTRING_END, "EOF", EXPR_LIT,
1166
1077
  :tNL, nil, EXPR_BEG)
1167
1078
  end
1168
1079
 
@@ -1173,51 +1084,10 @@ class TestRubyLexer < Minitest::Test
1173
1084
  :tEQL, "=", EXPR_BEG,
1174
1085
  :tSTRING_BEG, "\"", EXPR_BEG,
1175
1086
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1176
- :tSTRING_END, "EOF", EXPR_END,
1087
+ :tSTRING_END, "EOF", EXPR_LIT,
1177
1088
  :tNL, nil, EXPR_BEG)
1178
1089
  end
1179
1090
 
1180
- def test_yylex_heredoc_double_squiggly
1181
- setup_lexer_class Ruby23Parser
1182
-
1183
- assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1184
- nil,
1185
- :tIDENTIFIER, "a", EXPR_CMDARG,
1186
- :tEQL, "=", EXPR_BEG,
1187
- :tSTRING_BEG, "\"", EXPR_BEG,
1188
- :tSTRING_CONTENT, "blah blah\n", EXPR_BEG,
1189
- :tSTRING_END, "EOF", EXPR_END,
1190
- :tNL, nil, EXPR_BEG)
1191
- end
1192
-
1193
- # mri handles tabs in a pretty specific way:
1194
- # https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
1195
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
1196
- setup_lexer_class Ruby23Parser
1197
-
1198
- assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
1199
- nil,
1200
- :tIDENTIFIER, "a", EXPR_CMDARG,
1201
- :tEQL, "=", EXPR_BEG,
1202
- :tSTRING_BEG, "\"", EXPR_BEG,
1203
- :tSTRING_CONTENT, "blah blah\n\tblah blah\n", EXPR_BEG,
1204
- :tSTRING_END, "EOF", EXPR_END,
1205
- :tNL, nil, EXPR_BEG)
1206
- end
1207
-
1208
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
1209
- setup_lexer_class Ruby23Parser
1210
-
1211
- assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
1212
- nil,
1213
- :tIDENTIFIER, "a", EXPR_CMDARG,
1214
- :tEQL, "=", EXPR_BEG,
1215
- :tSTRING_BEG, "\"", EXPR_BEG,
1216
- :tSTRING_CONTENT, "blah blah\n blah blah\n", EXPR_BEG,
1217
- :tSTRING_END, "EOF", EXPR_END,
1218
- :tNL, nil, EXPR_BEG)
1219
- end
1220
-
1221
1091
  def test_yylex_heredoc_double_eos
1222
1092
  refute_lex("a = <<\"EOF\"\nblah",
1223
1093
  :tIDENTIFIER, "a",
@@ -1247,17 +1117,30 @@ class TestRubyLexer < Minitest::Test
1247
1117
  :tSTRING_CONTENT, "@@d ", EXPR_BEG, # HUH?
1248
1118
  :tSTRING_DBEG, "\#{", EXPR_BEG,
1249
1119
  :tSTRING_CONTENT, "3} \n", EXPR_BEG,
1250
- :tSTRING_END, "EOF", EXPR_END,
1120
+ :tSTRING_END, "EOF", EXPR_LIT,
1251
1121
  :tNL, nil, EXPR_BEG)
1252
1122
  end
1253
1123
 
1124
+ def test_yylex_heredoc_double_squiggly
1125
+ setup_lexer_class Ruby23Parser
1126
+
1127
+ assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1128
+ nil,
1129
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1130
+ :tEQL, "=", EXPR_BEG,
1131
+ :tSTRING_BEG, "\"", EXPR_BEG,
1132
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1133
+ :tSTRING_END, "EOF", EXPR_LIT,
1134
+ :tNL, nil, EXPR_BEG)
1135
+ end
1136
+
1254
1137
  def test_yylex_heredoc_empty
1255
1138
  assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1256
1139
  nil,
1257
1140
  :tSTRING_BEG, "\"", EXPR_BEG,
1258
1141
  :tSTRING_DBEG, "\#{", EXPR_BEG,
1259
1142
  :tSTRING_CONTENT, "x}\nblah2\n", EXPR_BEG,
1260
- :tSTRING_END, "", EXPR_END,
1143
+ :tSTRING_END, "", EXPR_LIT,
1261
1144
  :tNL, nil, EXPR_BEG)
1262
1145
  end
1263
1146
 
@@ -1268,7 +1151,7 @@ class TestRubyLexer < Minitest::Test
1268
1151
  :tEQL, "=", EXPR_BEG,
1269
1152
  :tSTRING_BEG, "\"", EXPR_BEG,
1270
1153
  :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1271
- :tSTRING_END, "EOF", EXPR_END,
1154
+ :tSTRING_END, "EOF", EXPR_LIT,
1272
1155
  :tNL, nil, EXPR_BEG)
1273
1156
  end
1274
1157
 
@@ -1286,7 +1169,7 @@ class TestRubyLexer < Minitest::Test
1286
1169
  :tEQL, "=", EXPR_BEG,
1287
1170
  :tSTRING_BEG, "\"", EXPR_BEG,
1288
1171
  :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1289
- :tSTRING_END, "EOF", EXPR_END,
1172
+ :tSTRING_END, "EOF", EXPR_LIT,
1290
1173
  :tNL, nil, EXPR_BEG)
1291
1174
  end
1292
1175
 
@@ -1298,8 +1181,8 @@ class TestRubyLexer < Minitest::Test
1298
1181
  :tIDENTIFIER, "a", EXPR_CMDARG,
1299
1182
  :tEQL, "=", EXPR_BEG,
1300
1183
  :tSTRING_BEG, "\"", EXPR_BEG,
1301
- :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1302
- :tSTRING_END, "EOF", EXPR_END,
1184
+ :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1185
+ :tSTRING_END, "EOF", EXPR_LIT,
1303
1186
  :tNL, nil, EXPR_BEG)
1304
1187
  end
1305
1188
 
@@ -1310,7 +1193,7 @@ class TestRubyLexer < Minitest::Test
1310
1193
  :tEQL, "=", EXPR_BEG,
1311
1194
  :tSTRING_BEG, "\"", EXPR_BEG,
1312
1195
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1313
- :tSTRING_END, "EOF", EXPR_END,
1196
+ :tSTRING_END, "EOF", EXPR_LIT,
1314
1197
  :tNL, nil, EXPR_BEG)
1315
1198
  end
1316
1199
 
@@ -1349,7 +1232,7 @@ class TestRubyLexer < Minitest::Test
1349
1232
  :tEQL, "=", EXPR_BEG,
1350
1233
  :tSTRING_BEG, "\"", EXPR_BEG,
1351
1234
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1352
- :tSTRING_END, "EOF", EXPR_END,
1235
+ :tSTRING_END, "EOF", EXPR_LIT,
1353
1236
  :tNL, nil, EXPR_BEG)
1354
1237
  end
1355
1238
 
@@ -1361,8 +1244,8 @@ class TestRubyLexer < Minitest::Test
1361
1244
  :tIDENTIFIER, "a", EXPR_CMDARG,
1362
1245
  :tEQL, "=", EXPR_BEG,
1363
1246
  :tSTRING_BEG, "\"", EXPR_BEG,
1364
- :tSTRING_CONTENT, "blah blah\n", EXPR_BEG,
1365
- :tSTRING_END, "EOF", EXPR_END,
1247
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1248
+ :tSTRING_END, "EOF", EXPR_LIT,
1366
1249
  :tNL, nil, EXPR_BEG)
1367
1250
  end
1368
1251
 
@@ -1392,41 +1275,41 @@ class TestRubyLexer < Minitest::Test
1392
1275
  assert_lex3("identifier?", nil, :tFID, "identifier?", EXPR_CMDARG)
1393
1276
  end
1394
1277
 
1395
- def test_yylex_identifier_equals_arrow
1396
- assert_lex3(":blah==>",
1397
- nil,
1398
- :tSYMBOL, "blah=", EXPR_END,
1399
- :tASSOC, "=>", EXPR_BEG)
1400
- end
1401
-
1402
1278
  def test_yylex_identifier_equals3
1403
1279
  assert_lex3(":a===b",
1404
1280
  nil,
1405
- :tSYMBOL, "a", EXPR_END,
1281
+ :tSYMBOL, "a", EXPR_LIT,
1406
1282
  :tEQQ, "===", EXPR_BEG,
1407
1283
  :tIDENTIFIER, "b", EXPR_ARG)
1408
1284
  end
1409
1285
 
1410
- def test_yylex_identifier_equals_equals_arrow
1411
- assert_lex3(":a==>b",
1286
+ def test_yylex_identifier_equals_arrow
1287
+ assert_lex3(":blah==>",
1412
1288
  nil,
1413
- :tSYMBOL, "a=", EXPR_END,
1414
- :tASSOC, "=>", EXPR_BEG,
1415
- :tIDENTIFIER, "b", EXPR_ARG)
1289
+ :tSYMBOL, "blah=", EXPR_LIT,
1290
+ :tASSOC, "=>", EXPR_BEG)
1416
1291
  end
1417
1292
 
1418
1293
  def test_yylex_identifier_equals_caret
1419
1294
  assert_lex_fname "^", :tCARET
1420
1295
  end
1421
1296
 
1297
+ def test_yylex_identifier_equals_def2
1298
+ assert_lex_fname "==", :tEQ
1299
+ end
1300
+
1422
1301
  def test_yylex_identifier_equals_def__20
1423
1302
  setup_lexer_class RubyParser::V20
1424
1303
 
1425
1304
  assert_lex_fname "identifier=", :tIDENTIFIER, EXPR_ENDFN
1426
1305
  end
1427
1306
 
1428
- def test_yylex_identifier_equals_def2
1429
- assert_lex_fname "==", :tEQ
1307
+ def test_yylex_identifier_equals_equals_arrow
1308
+ assert_lex3(":a==>b",
1309
+ nil,
1310
+ :tSYMBOL, "a=", EXPR_LIT,
1311
+ :tASSOC, "=>", EXPR_BEG,
1312
+ :tIDENTIFIER, "b", EXPR_ARG)
1430
1313
  end
1431
1314
 
1432
1315
  def test_yylex_identifier_equals_expr
@@ -1515,25 +1398,6 @@ class TestRubyLexer < Minitest::Test
1515
1398
  refute_lex "0d42__24"
1516
1399
  end
1517
1400
 
1518
- def test_yylex_integer_if_modifier
1519
- assert_lex3("123if",
1520
- nil,
1521
- :tINTEGER, 123, EXPR_NUM,
1522
- :kIF_MOD, "if", EXPR_PAR)
1523
- end
1524
-
1525
- def test_yylex_question_eh_a__20
1526
- setup_lexer_class RubyParser::V20
1527
-
1528
- assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
1529
- end
1530
-
1531
- def test_yylex_question_eh_escape_M_escape_C__20
1532
- setup_lexer_class RubyParser::V20
1533
-
1534
- assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
1535
- end
1536
-
1537
1401
  def test_yylex_integer_hex
1538
1402
  assert_lex3 "0x2a", nil, :tINTEGER, 42, EXPR_NUM
1539
1403
  end
@@ -1546,20 +1410,15 @@ class TestRubyLexer < Minitest::Test
1546
1410
  refute_lex "0xab__cd"
1547
1411
  end
1548
1412
 
1549
- def test_yylex_integer_oct
1550
- assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
1551
- end
1552
-
1553
- def test_yylex_integer_oct_bad_range
1554
- refute_lex "08"
1555
- end
1556
-
1557
- def test_yylex_integer_oct_bad_range2
1558
- refute_lex "08"
1413
+ def test_yylex_integer_if_modifier
1414
+ assert_lex3("123if",
1415
+ nil,
1416
+ :tINTEGER, 123, EXPR_NUM,
1417
+ :kIF_MOD, "if", EXPR_PAR)
1559
1418
  end
1560
1419
 
1561
- def test_yylex_integer_oct_bad_underscores
1562
- refute_lex "01__23"
1420
+ def test_yylex_integer_oct
1421
+ assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
1563
1422
  end
1564
1423
 
1565
1424
  def test_yylex_integer_oct_O
@@ -1578,6 +1437,18 @@ class TestRubyLexer < Minitest::Test
1578
1437
  assert_lex3 "0O ", nil, :tINTEGER, 0, EXPR_NUM
1579
1438
  end
1580
1439
 
1440
+ def test_yylex_integer_oct_bad_range
1441
+ refute_lex "08"
1442
+ end
1443
+
1444
+ def test_yylex_integer_oct_bad_range2
1445
+ refute_lex "08"
1446
+ end
1447
+
1448
+ def test_yylex_integer_oct_bad_underscores
1449
+ refute_lex "01__23"
1450
+ end
1451
+
1581
1452
  def test_yylex_integer_oct_o
1582
1453
  assert_lex3 "0o52", nil, :tINTEGER, 42, EXPR_NUM
1583
1454
  end
@@ -1590,46 +1461,247 @@ class TestRubyLexer < Minitest::Test
1590
1461
  refute_lex "0o1__23"
1591
1462
  end
1592
1463
 
1593
- def test_yylex_integer_oct_o_not_bad_none
1594
- assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
1464
+ def test_yylex_integer_oct_o_not_bad_none
1465
+ assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
1466
+ end
1467
+
1468
+ def test_yylex_integer_trailing
1469
+ assert_lex3("1.to_s",
1470
+ nil,
1471
+ :tINTEGER, 1, EXPR_NUM,
1472
+ :tDOT, ".", EXPR_DOT,
1473
+ :tIDENTIFIER, "to_s", EXPR_ARG)
1474
+ end
1475
+
1476
+ def test_yylex_integer_underscore
1477
+ assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
1478
+ end
1479
+
1480
+ def test_yylex_integer_underscore_bad
1481
+ refute_lex "4__2"
1482
+ end
1483
+
1484
+ def test_yylex_integer_zero
1485
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1486
+ end
1487
+
1488
+ def test_yylex_is_your_spacebar_broken?
1489
+ assert_lex3(":a!=:b",
1490
+ nil,
1491
+ :tSYMBOL, "a", EXPR_LIT,
1492
+ :tNEQ, "!=", EXPR_BEG,
1493
+ :tSYMBOL, "b", EXPR_LIT)
1494
+ end
1495
+
1496
+ def test_yylex_iter_array_curly
1497
+ # this will lex, but doesn't parse... don't freak out.
1498
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1499
+ nil,
1500
+
1501
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1502
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1503
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1504
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1505
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1506
+ :tRBRACK, "]", EXPR_END, 0, 0,
1507
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1508
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1509
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1510
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1511
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1512
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1513
+ :tRCURLY, "}", EXPR_END, 0, 0)
1514
+ end
1515
+
1516
+ def test_yylex_iter_array_curly__24
1517
+ setup_lexer_class RubyParser::V24
1518
+
1519
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1520
+ s(:iter,
1521
+ s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
1522
+ s(:args, :c, :d)),
1523
+
1524
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1525
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1526
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1527
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1528
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1529
+ :tRBRACK, "]", EXPR_ENDARG, 0, 0,
1530
+ :tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
1531
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1532
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1533
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1534
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1535
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1536
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1537
+ end
1538
+
1539
+ def test_yylex_ivar
1540
+ assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
1541
+ end
1542
+
1543
+ def test_yylex_ivar_bad
1544
+ refute_lex "@1"
1545
+ end
1546
+
1547
+ def test_yylex_ivar_bad_0_length
1548
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
1549
+ end
1550
+
1551
+ def test_yylex_keyword_expr
1552
+ self.lex_state = EXPR_ENDARG
1553
+
1554
+ assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
1555
+ end
1556
+
1557
+ def test_yylex_label
1558
+ assert_lex3("{a:",
1559
+ nil,
1560
+ :tLBRACE, "{", EXPR_PAR,
1561
+ :tLABEL, "a", EXPR_LAB)
1562
+ end
1563
+
1564
+ def test_yylex_label_in_params
1565
+ assert_lex3("foo(a:",
1566
+ nil,
1567
+ :tIDENTIFIER, "foo", EXPR_CMDARG,
1568
+ :tLPAREN2, "(", EXPR_PAR,
1569
+ :tLABEL, "a", EXPR_LAB)
1570
+ end
1571
+
1572
+ def test_yylex_lambda_args
1573
+ assert_lex("-> (a) { }",
1574
+ s(:iter, s(:lambda),
1575
+ s(:args, :a)),
1576
+
1577
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1578
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1579
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1580
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1581
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1582
+ :tRCURLY, "}", EXPR_END, 0, 0)
1583
+ end
1584
+
1585
+ def test_yylex_lambda_args__24
1586
+ setup_lexer_class RubyParser::V24
1587
+
1588
+ assert_lex("-> (a) { }",
1589
+ s(:iter, s(:lambda),
1590
+ s(:args, :a)),
1591
+
1592
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1593
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1594
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1595
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1596
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1597
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1598
+ end
1599
+
1600
+ def test_yylex_lambda_args_opt
1601
+ assert_lex("-> (a=nil) { }",
1602
+ s(:iter, s(:lambda),
1603
+ s(:args, s(:lasgn, :a, s(:nil)))),
1604
+
1605
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1606
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1607
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1608
+ :tEQL, "=", EXPR_BEG, 1, 0,
1609
+ :kNIL, "nil", EXPR_END, 1, 0,
1610
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1611
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1612
+ :tRCURLY, "}", EXPR_END, 0, 0)
1613
+ end
1614
+
1615
+ def test_yylex_lambda_args_opt__24
1616
+ setup_lexer_class RubyParser::V24
1617
+
1618
+ assert_lex("-> (a=nil) { }",
1619
+ s(:iter, s(:lambda),
1620
+ s(:args, s(:lasgn, :a, s(:nil)))),
1621
+
1622
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1623
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1624
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1625
+ :tEQL, "=", EXPR_BEG, 1, 0,
1626
+ :kNIL, "nil", EXPR_END, 1, 0,
1627
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1628
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1629
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1595
1630
  end
1596
1631
 
1597
- def test_yylex_integer_trailing
1598
- assert_lex3("1.to_s",
1632
+ def test_yylex_lambda_as_args_with_block
1633
+ assert_lex3("a -> do end do end",
1599
1634
  nil,
1600
- :tINTEGER, 1, EXPR_NUM,
1601
- :tDOT, ".", EXPR_DOT,
1602
- :tIDENTIFIER, "to_s", EXPR_ARG)
1635
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1636
+ :tLAMBDA, nil, EXPR_ENDFN,
1637
+ :kDO, "do", EXPR_BEG,
1638
+ :kEND, "end", EXPR_END,
1639
+ :kDO, "do", EXPR_BEG,
1640
+ :kEND, "end", EXPR_END)
1603
1641
  end
1604
1642
 
1605
- def test_yylex_integer_underscore
1606
- assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
1607
- end
1643
+ def test_yylex_lambda_hash
1644
+ assert_lex("-> (a={}) { }",
1645
+ s(:iter, s(:lambda),
1646
+ s(:args, s(:lasgn, :a, s(:hash)))),
1608
1647
 
1609
- def test_yylex_integer_underscore_bad
1610
- refute_lex "4__2"
1648
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1649
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1650
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1651
+ :tEQL, "=", EXPR_BEG, 1, 0,
1652
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1653
+ :tRCURLY, "}", EXPR_END, 1, 0,
1654
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1655
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1656
+ :tRCURLY, "}", EXPR_END, 0, 0)
1611
1657
  end
1612
1658
 
1613
- def test_yylex_integer_zero
1614
- assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1615
- end
1659
+ def test_yylex_lambda_hash__24
1660
+ setup_lexer_class RubyParser::V24
1616
1661
 
1617
- def test_yylex_ivar
1618
- assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
1619
- end
1662
+ assert_lex("-> (a={}) { }",
1663
+ s(:iter, s(:lambda),
1664
+ s(:args, s(:lasgn, :a, s(:hash)))),
1620
1665
 
1621
- def test_yylex_ivar_bad
1622
- refute_lex "@1"
1666
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1667
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1668
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1669
+ :tEQL, "=", EXPR_BEG, 1, 0,
1670
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1671
+ :tRCURLY, "}", EXPR_ENDARG, 1, 0,
1672
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1673
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1674
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1623
1675
  end
1624
1676
 
1625
- def test_yylex_ivar_bad_0_length
1626
- refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
1627
- end
1677
+ def test_yylex_lasgn_call_same_name
1678
+ assert_lex("a = b.c :d => 1",
1679
+ s(:lasgn, :a,
1680
+ s(:call, s(:call, nil, :b), :c,
1681
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1628
1682
 
1629
- def test_yylex_keyword_expr
1630
- self.lex_state = EXPR_ENDARG
1683
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1684
+ :tEQL, "=", EXPR_BEG, 0, 0,
1685
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1686
+ :tDOT, ".", EXPR_DOT, 0, 0,
1687
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
1688
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1689
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1690
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1631
1691
 
1632
- assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
1692
+ assert_lex("a = b.a :d => 1",
1693
+ s(:lasgn, :a,
1694
+ s(:call, s(:call, nil, :b), :a,
1695
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1696
+
1697
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1698
+ :tEQL, "=", EXPR_BEG, 0, 0,
1699
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1700
+ :tDOT, ".", EXPR_DOT, 0, 0,
1701
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
1702
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1703
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1704
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1633
1705
  end
1634
1706
 
1635
1707
  def test_yylex_lt
@@ -1656,12 +1728,22 @@ class TestRubyLexer < Minitest::Test
1656
1728
  assert_lex3("<=", nil, :tLEQ, "<=", EXPR_BEG)
1657
1729
  end
1658
1730
 
1731
+ def test_yylex_method_parens_chevron
1732
+ assert_lex("a()<<1",
1733
+ s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
1734
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1735
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1736
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1737
+ :tLSHFT, "<<", EXPR_BEG, 0, 0,
1738
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1739
+ end
1740
+
1659
1741
  def test_yylex_minus
1660
1742
  assert_lex3("1 - 2",
1661
1743
  nil,
1662
- :tINTEGER, 1, EXPR_NUM,
1744
+ :tINTEGER, 1, EXPR_NUM,
1663
1745
  :tMINUS, "-", EXPR_BEG,
1664
- :tINTEGER, 2, EXPR_NUM)
1746
+ :tINTEGER, 2, EXPR_NUM)
1665
1747
  end
1666
1748
 
1667
1749
  def test_yylex_minus_equals
@@ -1684,7 +1766,39 @@ class TestRubyLexer < Minitest::Test
1684
1766
  assert_lex3("-42",
1685
1767
  nil,
1686
1768
  :tUMINUS_NUM, "-", EXPR_BEG,
1687
- :tINTEGER, 42, EXPR_NUM)
1769
+ :tINTEGER, 42, EXPR_NUM)
1770
+ end
1771
+
1772
+ def test_yylex_not_at_defn
1773
+ assert_lex("def +@; end",
1774
+ s(:defn, :+@, s(:args), s(:nil)),
1775
+
1776
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1777
+ :tUPLUS, "+@", EXPR_ARG, 0, 0,
1778
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1779
+ :kEND, "end", EXPR_END, 0, 0)
1780
+
1781
+ assert_lex("def !@; end",
1782
+ s(:defn, :"!@", s(:args), s(:nil)),
1783
+
1784
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1785
+ :tUBANG, "!@", EXPR_ARG, 0, 0,
1786
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1787
+ :kEND, "end", EXPR_END, 0, 0)
1788
+ end
1789
+
1790
+ def test_yylex_not_at_ivar
1791
+ assert_lex("!@ivar",
1792
+ s(:call, s(:ivar, :@ivar), :"!"),
1793
+
1794
+ :tBANG, "!", EXPR_BEG, 0, 0,
1795
+ :tIVAR, "@ivar", EXPR_END, 0, 0)
1796
+ end
1797
+
1798
+ def test_yylex_not_unary_method
1799
+ self.lex_state = EXPR_FNAME
1800
+
1801
+ assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
1688
1802
  end
1689
1803
 
1690
1804
  def test_yylex_nth_ref
@@ -1700,7 +1814,68 @@ class TestRubyLexer < Minitest::Test
1700
1814
  :tNTH_REF, 7, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1701
1815
  :tNTH_REF, 8, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1702
1816
  :tNTH_REF, 9, EXPR_END,
1703
- :tRBRACK, "]", EXPR_ENDARG)
1817
+ :tRBRACK, "]", EXPR_END)
1818
+ end
1819
+
1820
+ def test_yylex_number_times_ident_times_return_number
1821
+ assert_lex("1 * b * 3",
1822
+ s(:call,
1823
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1824
+ :*, s(:lit, 3)),
1825
+
1826
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1827
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1828
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1829
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1830
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1831
+
1832
+ assert_lex("1 * b *\n 3",
1833
+ s(:call,
1834
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1835
+ :*, s(:lit, 3)),
1836
+
1837
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1838
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1839
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1840
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1841
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1842
+ end
1843
+
1844
+ def test_yylex_numbers
1845
+ assert_lex3 "0b10", nil, :tINTEGER, 2, EXPR_NUM
1846
+ assert_lex3 "0B10", nil, :tINTEGER, 2, EXPR_NUM
1847
+
1848
+ assert_lex3 "0d10", nil, :tINTEGER, 10, EXPR_NUM
1849
+ assert_lex3 "0D10", nil, :tINTEGER, 10, EXPR_NUM
1850
+
1851
+ assert_lex3 "0x10", nil, :tINTEGER, 16, EXPR_NUM
1852
+ assert_lex3 "0X10", nil, :tINTEGER, 16, EXPR_NUM
1853
+
1854
+ assert_lex3 "0o10", nil, :tINTEGER, 8, EXPR_NUM
1855
+ assert_lex3 "0O10", nil, :tINTEGER, 8, EXPR_NUM
1856
+
1857
+ assert_lex3 "0o", nil, :tINTEGER, 0, EXPR_NUM
1858
+ assert_lex3 "0O", nil, :tINTEGER, 0, EXPR_NUM
1859
+
1860
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1861
+
1862
+ refute_lex "0x"
1863
+ refute_lex "0X"
1864
+ refute_lex "0b"
1865
+ refute_lex "0B"
1866
+ refute_lex "0d"
1867
+ refute_lex "0D"
1868
+
1869
+ refute_lex "08"
1870
+ refute_lex "09"
1871
+ refute_lex "0o8"
1872
+ refute_lex "0o9"
1873
+ refute_lex "0O8"
1874
+ refute_lex "0O9"
1875
+
1876
+ refute_lex "1_e1"
1877
+ refute_lex "1_.1"
1878
+ refute_lex "1__1"
1704
1879
  end
1705
1880
 
1706
1881
  def test_yylex_open_bracket
@@ -1730,9 +1905,9 @@ class TestRubyLexer < Minitest::Test
1730
1905
  assert_lex3("m { 3 }",
1731
1906
  nil,
1732
1907
  :tIDENTIFIER, "m", EXPR_CMDARG,
1733
- :tLCURLY, "{", EXPR_PAR,
1734
- :tINTEGER, 3, EXPR_NUM,
1735
- :tRCURLY, "}", EXPR_ENDARG)
1908
+ :tLCURLY, "{", EXPR_PAR,
1909
+ :tINTEGER, 3, EXPR_NUM,
1910
+ :tRCURLY, "}", EXPR_END)
1736
1911
  end
1737
1912
 
1738
1913
  def test_yylex_open_curly_bracket_block
@@ -1741,8 +1916,8 @@ class TestRubyLexer < Minitest::Test
1741
1916
  assert_lex3("{ 4 }",
1742
1917
  nil,
1743
1918
  :tLBRACE_ARG, "{", EXPR_BEG,
1744
- :tINTEGER, 4, EXPR_NUM,
1745
- :tRCURLY, "}", EXPR_ENDARG)
1919
+ :tINTEGER, 4, EXPR_NUM,
1920
+ :tRCURLY, "}", EXPR_END)
1746
1921
  end
1747
1922
 
1748
1923
  def test_yylex_open_square_bracket_arg
@@ -1751,32 +1926,32 @@ class TestRubyLexer < Minitest::Test
1751
1926
  assert_lex3("m [ 3 ]",
1752
1927
  nil,
1753
1928
  :tIDENTIFIER, "m", EXPR_CMDARG,
1754
- :tLBRACK, "[", EXPR_PAR,
1755
- :tINTEGER, 3, EXPR_NUM,
1756
- :tRBRACK, "]", EXPR_ENDARG)
1929
+ :tLBRACK, "[", EXPR_PAR,
1930
+ :tINTEGER, 3, EXPR_NUM,
1931
+ :tRBRACK, "]", EXPR_END)
1757
1932
  end
1758
1933
 
1759
1934
  def test_yylex_open_square_bracket_ary
1760
1935
  assert_lex3("[1, 2, 3]",
1761
1936
  nil,
1762
1937
  :tLBRACK, "[", EXPR_PAR,
1763
- :tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1764
- :tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1938
+ :tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1939
+ :tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1765
1940
  :tINTEGER, 3, EXPR_NUM,
1766
- :tRBRACK, "]", EXPR_ENDARG)
1941
+ :tRBRACK, "]", EXPR_END)
1767
1942
  end
1768
1943
 
1769
1944
  def test_yylex_open_square_bracket_meth
1770
1945
  assert_lex3("m[3]",
1771
1946
  nil,
1772
1947
  :tIDENTIFIER, "m", EXPR_CMDARG,
1773
- :tLBRACK2, "[", EXPR_PAR,
1774
- :tINTEGER, 3, EXPR_NUM,
1775
- :tRBRACK, "]", EXPR_ENDARG)
1948
+ :tLBRACK2, "[", EXPR_PAR,
1949
+ :tINTEGER, 3, EXPR_NUM,
1950
+ :tRBRACK, "]", EXPR_END)
1776
1951
  end
1777
1952
 
1778
1953
  def test_yylex_or
1779
- assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
1954
+ assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
1780
1955
  end
1781
1956
 
1782
1957
  def test_yylex_or2
@@ -1791,12 +1966,85 @@ class TestRubyLexer < Minitest::Test
1791
1966
  assert_lex3("|=", nil, :tOP_ASGN, "|", EXPR_BEG)
1792
1967
  end
1793
1968
 
1969
+ def test_yylex_paren_string_interpolated_regexp
1970
+ setup_lexer('%( #{(/abcd/)} )',
1971
+ s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
1972
+
1973
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
1974
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1975
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1976
+
1977
+ emulate_string_interpolation do
1978
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
1979
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
1980
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
1981
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
1982
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
1983
+ end
1984
+
1985
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1986
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1987
+
1988
+ refute_lexeme
1989
+ end
1990
+
1991
+ def test_yylex_paren_string_parens_interpolated
1992
+ setup_lexer('%((#{b}#{d}))',
1993
+ s(:dstr,
1994
+ "(",
1995
+ s(:evstr, s(:call, nil, :b)),
1996
+ s(:evstr, s(:call, nil, :d)),
1997
+ s(:str, ")")))
1998
+
1999
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2000
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2001
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2002
+
2003
+ emulate_string_interpolation do
2004
+ assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2005
+ end
2006
+
2007
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2008
+
2009
+ emulate_string_interpolation do
2010
+ assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
2011
+ end
2012
+
2013
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2014
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
2015
+
2016
+ refute_lexeme
2017
+ end
2018
+
2019
+ def test_yylex_paren_string_parens_interpolated_regexp
2020
+ setup_lexer('%((#{(/abcd/)}))',
2021
+ s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
2022
+
2023
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2024
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2025
+
2026
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2027
+
2028
+ emulate_string_interpolation do
2029
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
2030
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
2031
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
2032
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
2033
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
2034
+ end
2035
+
2036
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2037
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
2038
+
2039
+ refute_lexeme
2040
+ end
2041
+
1794
2042
  def test_yylex_percent
1795
2043
  assert_lex3("a % 2",
1796
2044
  nil,
1797
2045
  :tIDENTIFIER, "a", EXPR_CMDARG,
1798
2046
  :tPERCENT, "%", EXPR_BEG,
1799
- :tINTEGER, 2, EXPR_NUM)
2047
+ :tINTEGER, 2, EXPR_NUM)
1800
2048
  end
1801
2049
 
1802
2050
  def test_yylex_percent_equals
@@ -1804,15 +2052,15 @@ class TestRubyLexer < Minitest::Test
1804
2052
  nil,
1805
2053
  :tIDENTIFIER, "a", EXPR_CMDARG,
1806
2054
  :tOP_ASGN, "%", EXPR_BEG,
1807
- :tINTEGER, 2, EXPR_NUM)
2055
+ :tINTEGER, 2, EXPR_NUM)
1808
2056
  end
1809
2057
 
1810
2058
  def test_yylex_plus
1811
2059
  assert_lex3("1 + 1", # TODO lex_state?
1812
2060
  nil,
1813
- :tINTEGER, 1, EXPR_NUM,
2061
+ :tINTEGER, 1, EXPR_NUM,
1814
2062
  :tPLUS, "+", EXPR_BEG,
1815
- :tINTEGER, 1, EXPR_NUM)
2063
+ :tINTEGER, 1, EXPR_NUM)
1816
2064
  end
1817
2065
 
1818
2066
  def test_yylex_plus_equals
@@ -1831,55 +2079,24 @@ class TestRubyLexer < Minitest::Test
1831
2079
  assert_lex3("+@", nil, :tUPLUS, "+@", EXPR_ARG)
1832
2080
  end
1833
2081
 
1834
- def test_yylex_not_unary_method
1835
- self.lex_state = EXPR_FNAME
1836
-
1837
- assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
2082
+ def test_yylex_plus_unary_number
2083
+ assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
1838
2084
  end
1839
2085
 
1840
- def test_yylex_numbers
1841
- assert_lex3("0b10", nil, :tINTEGER, 2, EXPR_NUM)
1842
- assert_lex3("0B10", nil, :tINTEGER, 2, EXPR_NUM)
1843
-
1844
- assert_lex3("0d10", nil, :tINTEGER, 10, EXPR_NUM)
1845
- assert_lex3("0D10", nil, :tINTEGER, 10, EXPR_NUM)
1846
-
1847
- assert_lex3("0x10", nil, :tINTEGER, 16, EXPR_NUM)
1848
- assert_lex3("0X10", nil, :tINTEGER, 16, EXPR_NUM)
1849
-
1850
- assert_lex3("0o10", nil, :tINTEGER, 8, EXPR_NUM)
1851
- assert_lex3("0O10", nil, :tINTEGER, 8, EXPR_NUM)
1852
-
1853
- assert_lex3("0o", nil, :tINTEGER, 0, EXPR_NUM)
1854
- assert_lex3("0O", nil, :tINTEGER, 0, EXPR_NUM)
1855
-
1856
- assert_lex3("0", nil, :tINTEGER, 0, EXPR_NUM)
1857
-
1858
- refute_lex "0x"
1859
- refute_lex "0X"
1860
- refute_lex "0b"
1861
- refute_lex "0B"
1862
- refute_lex "0d"
1863
- refute_lex "0D"
2086
+ def test_yylex_question_bad_eos
2087
+ refute_lex "?"
2088
+ end
1864
2089
 
1865
- refute_lex "08"
1866
- refute_lex "09"
1867
- refute_lex "0o8"
1868
- refute_lex "0o9"
1869
- refute_lex "0O8"
1870
- refute_lex "0O9"
2090
+ def test_yylex_question_eh_a__20
2091
+ setup_lexer_class RubyParser::V20
1871
2092
 
1872
- refute_lex "1_e1"
1873
- refute_lex "1_.1"
1874
- refute_lex "1__1"
2093
+ assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
1875
2094
  end
1876
2095
 
1877
- def test_yylex_plus_unary_number
1878
- assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
1879
- end
2096
+ def test_yylex_question_eh_escape_M_escape_C__20
2097
+ setup_lexer_class RubyParser::V20
1880
2098
 
1881
- def test_yylex_question_bad_eos
1882
- refute_lex "?"
2099
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
1883
2100
  end
1884
2101
 
1885
2102
  def test_yylex_question_ws
@@ -1903,11 +2120,11 @@ class TestRubyLexer < Minitest::Test
1903
2120
  end
1904
2121
 
1905
2122
  def test_yylex_rbracket
1906
- assert_lex3("]", nil, :tRBRACK, "]", EXPR_ENDARG)
2123
+ assert_lex3("]", nil, :tRBRACK, "]", EXPR_END)
1907
2124
  end
1908
2125
 
1909
2126
  def test_yylex_rcurly
1910
- assert_lex("}", nil, :tRCURLY, "}", EXPR_ENDARG, 0, 1) do
2127
+ assert_lex("}", nil, :tRCURLY, "}", EXPR_END, 0, 1) do
1911
2128
  lexer.brace_nest += 2
1912
2129
  end
1913
2130
  end
@@ -1917,7 +2134,7 @@ class TestRubyLexer < Minitest::Test
1917
2134
  nil,
1918
2135
  :tREGEXP_BEG, "/", EXPR_BEG,
1919
2136
  :tSTRING_CONTENT, "regexp", EXPR_BEG,
1920
- :tREGEXP_END, "", EXPR_END)
2137
+ :tREGEXP_END, "", EXPR_LIT)
1921
2138
  end
1922
2139
 
1923
2140
  def test_yylex_regexp_ambiguous
@@ -1926,7 +2143,7 @@ class TestRubyLexer < Minitest::Test
1926
2143
  :tIDENTIFIER, "method", EXPR_CMDARG,
1927
2144
  :tREGEXP_BEG, "/", EXPR_CMDARG,
1928
2145
  :tSTRING_CONTENT, "regexp", EXPR_CMDARG,
1929
- :tREGEXP_END, "", EXPR_END)
2146
+ :tREGEXP_END, "", EXPR_LIT)
1930
2147
  end
1931
2148
 
1932
2149
  def test_yylex_regexp_bad
@@ -1940,7 +2157,7 @@ class TestRubyLexer < Minitest::Test
1940
2157
  nil,
1941
2158
  :tREGEXP_BEG, "/", EXPR_BEG,
1942
2159
  :tSTRING_CONTENT, "regex\\C-x", EXPR_BEG,
1943
- :tREGEXP_END, "", EXPR_END)
2160
+ :tREGEXP_END, "", EXPR_LIT)
1944
2161
  end
1945
2162
 
1946
2163
  def test_yylex_regexp_escape_C_M
@@ -1948,7 +2165,7 @@ class TestRubyLexer < Minitest::Test
1948
2165
  nil,
1949
2166
  :tREGEXP_BEG, "/", EXPR_BEG,
1950
2167
  :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
1951
- :tREGEXP_END, "", EXPR_END)
2168
+ :tREGEXP_END, "", EXPR_LIT)
1952
2169
  end
1953
2170
 
1954
2171
  def test_yylex_regexp_escape_C_M_craaaazy
@@ -1956,7 +2173,7 @@ class TestRubyLexer < Minitest::Test
1956
2173
  nil,
1957
2174
  :tREGEXP_BEG, "/", EXPR_BEG,
1958
2175
  :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
1959
- :tREGEXP_END, "", EXPR_END)
2176
+ :tREGEXP_END, "", EXPR_LIT)
1960
2177
  end
1961
2178
 
1962
2179
  def test_yylex_regexp_escape_C_bad_dash
@@ -1984,7 +2201,7 @@ class TestRubyLexer < Minitest::Test
1984
2201
  nil,
1985
2202
  :tREGEXP_BEG, "/", EXPR_BEG,
1986
2203
  :tSTRING_CONTENT, "regex\\M-x", EXPR_BEG,
1987
- :tREGEXP_END, "", EXPR_END)
2204
+ :tREGEXP_END, "", EXPR_LIT)
1988
2205
  end
1989
2206
 
1990
2207
  def test_yylex_regexp_escape_M_C
@@ -1992,7 +2209,7 @@ class TestRubyLexer < Minitest::Test
1992
2209
  nil,
1993
2210
  :tREGEXP_BEG, "/", EXPR_BEG,
1994
2211
  :tSTRING_CONTENT, "regex\\M-\\C-x", EXPR_BEG,
1995
- :tREGEXP_END, "", EXPR_END)
2212
+ :tREGEXP_END, "", EXPR_LIT)
1996
2213
  end
1997
2214
 
1998
2215
  def test_yylex_regexp_escape_M_bad_dash
@@ -2016,7 +2233,7 @@ class TestRubyLexer < Minitest::Test
2016
2233
  nil,
2017
2234
  :tREGEXP_BEG, "/", EXPR_BEG,
2018
2235
  :tSTRING_CONTENT, "\\/", EXPR_BEG,
2019
- :tREGEXP_END, "", EXPR_END)
2236
+ :tREGEXP_END, "", EXPR_LIT)
2020
2237
  end
2021
2238
 
2022
2239
  def test_yylex_regexp_escape_backslash_terminator
@@ -2024,15 +2241,7 @@ class TestRubyLexer < Minitest::Test
2024
2241
  nil,
2025
2242
  :tREGEXP_BEG, "%r\000", EXPR_BEG,
2026
2243
  :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2027
- :tREGEXP_END, "", EXPR_END)
2028
- end
2029
-
2030
- def test_yylex_regexp_escaped_delim
2031
- assert_lex3("%r!blah(?\\!blah)!",
2032
- nil,
2033
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2034
- :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2035
- :tREGEXP_END, "", EXPR_END)
2244
+ :tREGEXP_END, "", EXPR_LIT)
2036
2245
  end
2037
2246
 
2038
2247
  def test_yylex_regexp_escape_backslash_terminator_meta1
@@ -2040,7 +2249,7 @@ class TestRubyLexer < Minitest::Test
2040
2249
  nil,
2041
2250
  :tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
2042
2251
  :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2043
- :tREGEXP_END, "", EXPR_END)
2252
+ :tREGEXP_END, "", EXPR_LIT)
2044
2253
  end
2045
2254
 
2046
2255
  def test_yylex_regexp_escape_backslash_terminator_meta2
@@ -2048,7 +2257,7 @@ class TestRubyLexer < Minitest::Test
2048
2257
  nil,
2049
2258
  :tREGEXP_BEG, "%r\000", EXPR_BEG,
2050
2259
  :tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
2051
- :tREGEXP_END, "", EXPR_END)
2260
+ :tREGEXP_END, "", EXPR_LIT)
2052
2261
  end
2053
2262
 
2054
2263
  def test_yylex_regexp_escape_backslash_terminator_meta3
@@ -2056,7 +2265,7 @@ class TestRubyLexer < Minitest::Test
2056
2265
  nil,
2057
2266
  :tREGEXP_BEG, "%r\000", EXPR_BEG,
2058
2267
  :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2059
- :tREGEXP_END, "", EXPR_END)
2268
+ :tREGEXP_END, "", EXPR_LIT)
2060
2269
  end
2061
2270
 
2062
2271
  def test_yylex_regexp_escape_bad_eos
@@ -2068,7 +2277,7 @@ class TestRubyLexer < Minitest::Test
2068
2277
  nil,
2069
2278
  :tREGEXP_BEG, "/", EXPR_BEG,
2070
2279
  :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2071
- :tREGEXP_END, "", EXPR_END)
2280
+ :tREGEXP_END, "", EXPR_LIT)
2072
2281
  end
2073
2282
 
2074
2283
  def test_yylex_regexp_escape_c
@@ -2076,7 +2285,7 @@ class TestRubyLexer < Minitest::Test
2076
2285
  nil,
2077
2286
  :tREGEXP_BEG, "/", EXPR_BEG,
2078
2287
  :tSTRING_CONTENT, "regex\\cxxx", EXPR_BEG,
2079
- :tREGEXP_END, "", EXPR_END)
2288
+ :tREGEXP_END, "", EXPR_LIT)
2080
2289
  end
2081
2290
 
2082
2291
  def test_yylex_regexp_escape_c_backslash
@@ -2084,7 +2293,7 @@ class TestRubyLexer < Minitest::Test
2084
2293
  nil,
2085
2294
  :tREGEXP_BEG, "/", EXPR_BEG,
2086
2295
  :tSTRING_CONTENT, "regex\\c\\n", EXPR_BEG,
2087
- :tREGEXP_END, "", EXPR_END)
2296
+ :tREGEXP_END, "", EXPR_LIT)
2088
2297
  end
2089
2298
 
2090
2299
  def test_yylex_regexp_escape_chars
@@ -2092,7 +2301,7 @@ class TestRubyLexer < Minitest::Test
2092
2301
  nil,
2093
2302
  :tREGEXP_BEG, "/", EXPR_BEG,
2094
2303
  :tSTRING_CONTENT, "re\\tge\\nxp", EXPR_BEG,
2095
- :tREGEXP_END, "", EXPR_END)
2304
+ :tREGEXP_END, "", EXPR_LIT)
2096
2305
  end
2097
2306
 
2098
2307
  def test_yylex_regexp_escape_double_backslash
@@ -2101,7 +2310,7 @@ class TestRubyLexer < Minitest::Test
2101
2310
  nil,
2102
2311
  :tREGEXP_BEG, "/", EXPR_BEG,
2103
2312
  :tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
2104
- :tREGEXP_END, "", EXPR_END)
2313
+ :tREGEXP_END, "", EXPR_LIT)
2105
2314
  end
2106
2315
 
2107
2316
  def test_yylex_regexp_escape_hex
@@ -2109,7 +2318,7 @@ class TestRubyLexer < Minitest::Test
2109
2318
  nil,
2110
2319
  :tREGEXP_BEG, "/", EXPR_BEG,
2111
2320
  :tSTRING_CONTENT, "regex\\x61xp", EXPR_BEG,
2112
- :tREGEXP_END, "", EXPR_END)
2321
+ :tREGEXP_END, "", EXPR_LIT)
2113
2322
  end
2114
2323
 
2115
2324
  def test_yylex_regexp_escape_hex_bad
@@ -2121,7 +2330,7 @@ class TestRubyLexer < Minitest::Test
2121
2330
  nil,
2122
2331
  :tREGEXP_BEG, "/", EXPR_BEG,
2123
2332
  :tSTRING_CONTENT, "^[\\xd\\xa]{2}", EXPR_BEG,
2124
- :tREGEXP_END, "on", EXPR_END)
2333
+ :tREGEXP_END, "on", EXPR_LIT)
2125
2334
  end
2126
2335
 
2127
2336
  def test_yylex_regexp_escape_oct1
@@ -2129,7 +2338,7 @@ class TestRubyLexer < Minitest::Test
2129
2338
  nil,
2130
2339
  :tREGEXP_BEG, "/", EXPR_BEG,
2131
2340
  :tSTRING_CONTENT, "regex\\0xp", EXPR_BEG,
2132
- :tREGEXP_END, "", EXPR_END)
2341
+ :tREGEXP_END, "", EXPR_LIT)
2133
2342
  end
2134
2343
 
2135
2344
  def test_yylex_regexp_escape_oct2
@@ -2137,7 +2346,7 @@ class TestRubyLexer < Minitest::Test
2137
2346
  nil,
2138
2347
  :tREGEXP_BEG, "/", EXPR_BEG,
2139
2348
  :tSTRING_CONTENT, "regex\\07xp", EXPR_BEG,
2140
- :tREGEXP_END, "", EXPR_END)
2349
+ :tREGEXP_END, "", EXPR_LIT)
2141
2350
  end
2142
2351
 
2143
2352
  def test_yylex_regexp_escape_oct3
@@ -2145,15 +2354,23 @@ class TestRubyLexer < Minitest::Test
2145
2354
  nil,
2146
2355
  :tREGEXP_BEG, "/", EXPR_BEG,
2147
2356
  :tSTRING_CONTENT, "regex\\10142", EXPR_BEG,
2148
- :tREGEXP_END, "", EXPR_END)
2357
+ :tREGEXP_END, "", EXPR_LIT)
2149
2358
  end
2150
2359
 
2151
2360
  def test_yylex_regexp_escape_return
2152
2361
  assert_lex3("/regex\\\nregex/",
2153
2362
  nil,
2154
- :tREGEXP_BEG, "/", EXPR_BEG,
2155
- :tSTRING_CONTENT, "regexregex", EXPR_BEG,
2156
- :tREGEXP_END, "", EXPR_END)
2363
+ :tREGEXP_BEG, "/", EXPR_BEG,
2364
+ :tSTRING_CONTENT, "regexregex", EXPR_BEG,
2365
+ :tREGEXP_END, "", EXPR_LIT)
2366
+ end
2367
+
2368
+ def test_yylex_regexp_escaped_delim
2369
+ assert_lex3("%r!blah(?\\!blah)!",
2370
+ nil,
2371
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2372
+ :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2373
+ :tREGEXP_END, "", EXPR_LIT)
2157
2374
  end
2158
2375
 
2159
2376
  def test_yylex_regexp_nm
@@ -2161,7 +2378,20 @@ class TestRubyLexer < Minitest::Test
2161
2378
  nil,
2162
2379
  :tREGEXP_BEG, "/", EXPR_BEG,
2163
2380
  :tSTRING_CONTENT, ".*", EXPR_BEG,
2164
- :tREGEXP_END, "nm", EXPR_END)
2381
+ :tREGEXP_END, "nm", EXPR_LIT)
2382
+ end
2383
+
2384
+ def test_yylex_required_kwarg_no_value_22
2385
+ setup_lexer_class RubyParser::V22
2386
+
2387
+ assert_lex3("def foo a:, b:\nend",
2388
+ nil,
2389
+ :kDEF, "def", EXPR_FNAME,
2390
+ :tIDENTIFIER, "foo", EXPR_ENDFN,
2391
+ :tLABEL, "a", EXPR_LAB,
2392
+ :tCOMMA, ",", EXPR_PAR,
2393
+ :tLABEL, "b", EXPR_LAB,
2394
+ :kEND, "end", EXPR_END)
2165
2395
  end
2166
2396
 
2167
2397
  def test_yylex_rparen
@@ -2173,7 +2403,7 @@ class TestRubyLexer < Minitest::Test
2173
2403
  nil,
2174
2404
  :tIDENTIFIER, "a", EXPR_CMDARG,
2175
2405
  :tRSHFT, ">>", EXPR_BEG,
2176
- :tINTEGER, 2, EXPR_NUM)
2406
+ :tINTEGER, 2, EXPR_NUM)
2177
2407
  end
2178
2408
 
2179
2409
  def test_yylex_rshft_equals
@@ -2181,7 +2411,7 @@ class TestRubyLexer < Minitest::Test
2181
2411
  nil,
2182
2412
  :tIDENTIFIER, "a", EXPR_CMDARG,
2183
2413
  :tOP_ASGN, ">>", EXPR_BEG,
2184
- :tINTEGER, 2, EXPR_NUM)
2414
+ :tINTEGER, 2, EXPR_NUM)
2185
2415
  end
2186
2416
 
2187
2417
  def test_yylex_star
@@ -2249,11 +2479,11 @@ class TestRubyLexer < Minitest::Test
2249
2479
  end
2250
2480
 
2251
2481
  def test_yylex_string_bad_eos
2252
- refute_lex('%', :tSTRING_BEG, '%')
2482
+ refute_lex("%", :tSTRING_BEG, "%")
2253
2483
  end
2254
2484
 
2255
2485
  def test_yylex_string_bad_eos_quote
2256
- refute_lex('%{nest', :tSTRING_BEG, '%}')
2486
+ refute_lex("%{nest", :tSTRING_BEG, "%}")
2257
2487
  end
2258
2488
 
2259
2489
  def test_yylex_string_double
@@ -2269,7 +2499,7 @@ class TestRubyLexer < Minitest::Test
2269
2499
  nil,
2270
2500
  :tSTRING_BEG, "\"", EXPR_BEG,
2271
2501
  :tSTRING_CONTENT, "\034", EXPR_BEG,
2272
- :tSTRING_END, "\"", EXPR_END)
2502
+ :tSTRING_END, "\"", EXPR_LIT)
2273
2503
  end
2274
2504
 
2275
2505
  def test_yylex_string_double_escape_C_escape
@@ -2277,106 +2507,25 @@ class TestRubyLexer < Minitest::Test
2277
2507
  nil,
2278
2508
  :tSTRING_BEG, "\"", EXPR_BEG,
2279
2509
  :tSTRING_CONTENT, "\201", EXPR_BEG,
2280
- :tSTRING_END, "\"", EXPR_END)
2510
+ :tSTRING_END, "\"", EXPR_LIT)
2281
2511
  end
2282
2512
 
2283
2513
  def test_yylex_string_double_escape_C_question
2284
2514
  assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", EXPR_END)
2285
2515
  end
2286
2516
 
2287
- def test_yylex_string_utf8_simple
2288
- chr = [0x3024].pack("U")
2289
-
2290
- assert_lex3('"\u{3024}"',
2291
- s(:str, chr),
2292
- :tSTRING, chr, EXPR_END)
2293
- end
2294
-
2295
- def test_yylex_string_utf8_trailing_hex
2296
- chr = [0x3024].pack("U")
2297
- str = "#{chr}abz"
2298
-
2299
- assert_lex3('"\u3024abz"',
2300
- s(:str, str),
2301
- :tSTRING, str, EXPR_END)
2302
- end
2303
-
2304
- def test_yylex_string_utf8_missing_hex
2305
- refute_lex('"\u3zzz"')
2306
- refute_lex('"\u30zzz"')
2307
- refute_lex('"\u302zzz"')
2308
- end
2309
-
2310
- def test_yylex_string_utf8_complex
2311
- chr = [0x3024].pack("U")
2312
-
2313
- assert_lex3('"#@a\u{3024}"',
2314
- s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2315
- :tSTRING_BEG, '"', EXPR_BEG,
2316
- :tSTRING_DVAR, nil, EXPR_BEG,
2317
- :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2318
- :tSTRING_END, '"', EXPR_END)
2319
- end
2320
-
2321
- def test_yylex_string_utf8_complex_trailing_hex
2322
- chr = [0x3024].pack("U")
2323
- str = "#{chr}abz"
2324
-
2325
- assert_lex3('"#@a\u3024abz"',
2326
- s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2327
- :tSTRING_BEG, '"', EXPR_BEG,
2328
- :tSTRING_DVAR, nil, EXPR_BEG,
2329
- :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2330
- :tSTRING_END, '"', EXPR_END)
2331
- end
2332
-
2333
- def test_yylex_string_utf8_complex_missing_hex
2334
- chr = [0x302].pack("U")
2335
- str = "#{chr}zzz"
2336
-
2337
- refute_lex('"#@a\u302zzz"',
2338
- :tSTRING_BEG, '"',
2339
- :tSTRING_DVAR, nil,
2340
- :tSTRING_CONTENT, "@a"+str,
2341
- :tSTRING_END, '"')
2342
-
2343
- chr = [0x30].pack("U")
2344
- str = "#{chr}zzz"
2345
-
2346
- refute_lex('"#@a\u30zzz"',
2347
- :tSTRING_BEG, '"',
2348
- :tSTRING_DVAR, nil,
2349
- :tSTRING_CONTENT, "@a"+str,
2350
- :tSTRING_END, '"')
2351
-
2352
- chr = [0x3].pack("U")
2353
- str = "#{chr}zzz"
2354
-
2355
- refute_lex('"#@a\u3zzz"',
2356
- :tSTRING_BEG, '"',
2357
- :tSTRING_DVAR, nil,
2358
- :tSTRING_CONTENT, "@a"+str,
2359
- :tSTRING_END, '"')
2360
- end
2361
-
2362
2517
  def test_yylex_string_double_escape_M
2363
2518
  chr = "\341"
2364
2519
 
2365
2520
  assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, EXPR_END)
2366
2521
  end
2367
2522
 
2368
- def test_why_does_ruby_hate_me?
2369
- assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2370
- nil,
2371
- :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, EXPR_END)
2372
- end
2373
-
2374
2523
  def test_yylex_string_double_escape_M_backslash
2375
2524
  assert_lex3("\"\\M-\\\\\"",
2376
2525
  nil,
2377
2526
  :tSTRING_BEG, "\"", EXPR_BEG,
2378
2527
  :tSTRING_CONTENT, "\334", EXPR_BEG,
2379
- :tSTRING_END, "\"", EXPR_END)
2528
+ :tSTRING_END, "\"", EXPR_LIT)
2380
2529
  end
2381
2530
 
2382
2531
  def test_yylex_string_double_escape_M_escape
@@ -2384,7 +2533,7 @@ class TestRubyLexer < Minitest::Test
2384
2533
  nil,
2385
2534
  :tSTRING_BEG, "\"", EXPR_BEG,
2386
2535
  :tSTRING_CONTENT, "\201", EXPR_BEG,
2387
- :tSTRING_END, "\"", EXPR_END)
2536
+ :tSTRING_END, "\"", EXPR_LIT)
2388
2537
  end
2389
2538
 
2390
2539
  def test_yylex_string_double_escape_bs1
@@ -2404,7 +2553,7 @@ class TestRubyLexer < Minitest::Test
2404
2553
  nil,
2405
2554
  :tSTRING_BEG, "\"", EXPR_BEG,
2406
2555
  :tSTRING_CONTENT, "\034", EXPR_BEG,
2407
- :tSTRING_END, "\"", EXPR_END)
2556
+ :tSTRING_END, "\"", EXPR_LIT)
2408
2557
  end
2409
2558
 
2410
2559
  def test_yylex_string_double_escape_c_escape
@@ -2412,7 +2561,7 @@ class TestRubyLexer < Minitest::Test
2412
2561
  nil,
2413
2562
  :tSTRING_BEG, "\"", EXPR_BEG,
2414
2563
  :tSTRING_CONTENT, "\201", EXPR_BEG,
2415
- :tSTRING_END, "\"", EXPR_END)
2564
+ :tSTRING_END, "\"", EXPR_LIT)
2416
2565
  end
2417
2566
 
2418
2567
  def test_yylex_string_double_escape_c_question
@@ -2446,15 +2595,7 @@ class TestRubyLexer < Minitest::Test
2446
2595
  :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2447
2596
  :tSTRING_DBEG, nil, EXPR_BEG,
2448
2597
  :tSTRING_CONTENT, "3} # ", EXPR_BEG,
2449
- :tSTRING_END, "\"", EXPR_END)
2450
- end
2451
-
2452
- def test_yylex_string_double_pound_dollar_bad
2453
- assert_lex3('"#$%"', nil,
2454
-
2455
- :tSTRING_BEG, "\"", EXPR_BEG,
2456
- :tSTRING_CONTENT, '#$%', EXPR_BEG,
2457
- :tSTRING_END, "\"", EXPR_END)
2598
+ :tSTRING_END, "\"", EXPR_LIT)
2458
2599
  end
2459
2600
 
2460
2601
  def test_yylex_string_double_nested_curlies
@@ -2462,7 +2603,7 @@ class TestRubyLexer < Minitest::Test
2462
2603
  nil,
2463
2604
  :tSTRING_BEG, "%}", EXPR_BEG,
2464
2605
  :tSTRING_CONTENT, "nest{one{two}one}nest", EXPR_BEG,
2465
- :tSTRING_END, "}", EXPR_END)
2606
+ :tSTRING_END, "}", EXPR_LIT)
2466
2607
  end
2467
2608
 
2468
2609
  def test_yylex_string_double_no_interp
@@ -2470,25 +2611,33 @@ class TestRubyLexer < Minitest::Test
2470
2611
  assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", EXPR_END)
2471
2612
  end
2472
2613
 
2614
+ def test_yylex_string_double_pound_dollar_bad
2615
+ assert_lex3('"#$%"', nil,
2616
+
2617
+ :tSTRING_BEG, "\"", EXPR_BEG,
2618
+ :tSTRING_CONTENT, "#\$%", EXPR_BEG,
2619
+ :tSTRING_END, "\"", EXPR_LIT)
2620
+ end
2621
+
2473
2622
  def test_yylex_string_escape_x_single
2474
2623
  assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", EXPR_END)
2475
2624
  end
2476
2625
 
2477
- def test_yylex_string_pct_i
2478
- assert_lex3("%i[s1 s2\ns3]",
2626
+ def test_yylex_string_pct_I
2627
+ assert_lex3("%I[s1 s2\ns3]",
2479
2628
  nil,
2480
- :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2629
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2481
2630
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2482
2631
  :tSPACE, nil, EXPR_BEG,
2483
2632
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2484
2633
  :tSPACE, nil, EXPR_BEG,
2485
2634
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2486
2635
  :tSPACE, nil, EXPR_BEG,
2487
- :tSTRING_END, nil, EXPR_END)
2636
+ :tSTRING_END, nil, EXPR_LIT)
2488
2637
  end
2489
2638
 
2490
- def test_yylex_string_pct_I
2491
- assert_lex3("%I[s1 s2\ns3]",
2639
+ def test_yylex_string_pct_I_extra_space
2640
+ assert_lex3("%I[ s1 s2\ns3 ]",
2492
2641
  nil,
2493
2642
  :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2494
2643
  :tSTRING_CONTENT, "s1", EXPR_BEG,
@@ -2497,11 +2646,51 @@ class TestRubyLexer < Minitest::Test
2497
2646
  :tSPACE, nil, EXPR_BEG,
2498
2647
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2499
2648
  :tSPACE, nil, EXPR_BEG,
2500
- :tSTRING_END, nil, EXPR_END)
2649
+ :tSTRING_END, nil, EXPR_LIT)
2501
2650
  end
2502
2651
 
2503
- def test_yylex_string_pct_i_extra_space
2504
- assert_lex3("%i[ s1 s2\ns3 ]",
2652
+ def test_yylex_string_pct_Q
2653
+ assert_lex3("%Q[s1 s2]",
2654
+ nil,
2655
+ :tSTRING_BEG, "%Q[", EXPR_BEG,
2656
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2657
+ :tSTRING_END, "]", EXPR_LIT)
2658
+ end
2659
+
2660
+ def test_yylex_string_pct_W
2661
+ assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2662
+ nil,
2663
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2664
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2665
+ :tSPACE, nil, EXPR_BEG,
2666
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2667
+ :tSPACE, nil, EXPR_BEG,
2668
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2669
+ :tSPACE, nil, EXPR_BEG,
2670
+ :tSTRING_END, nil, EXPR_LIT)
2671
+ end
2672
+
2673
+ def test_yylex_string_pct_W_bs_nl
2674
+ assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2675
+ nil,
2676
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2677
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2678
+ :tSPACE, nil, EXPR_BEG,
2679
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2680
+ :tSPACE, nil, EXPR_BEG,
2681
+ :tSTRING_END, nil, EXPR_LIT)
2682
+ end
2683
+
2684
+ def test_yylex_string_pct_angle
2685
+ assert_lex3("%<blah>",
2686
+ nil,
2687
+ :tSTRING_BEG, "%>", EXPR_BEG,
2688
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2689
+ :tSTRING_END, ">", EXPR_LIT)
2690
+ end
2691
+
2692
+ def test_yylex_string_pct_i
2693
+ assert_lex3("%i[s1 s2\ns3]",
2505
2694
  nil,
2506
2695
  :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2507
2696
  :tSTRING_CONTENT, "s1", EXPR_BEG,
@@ -2510,36 +2699,36 @@ class TestRubyLexer < Minitest::Test
2510
2699
  :tSPACE, nil, EXPR_BEG,
2511
2700
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2512
2701
  :tSPACE, nil, EXPR_BEG,
2513
- :tSTRING_END, nil, EXPR_END)
2702
+ :tSTRING_END, nil, EXPR_LIT)
2514
2703
  end
2515
2704
 
2516
- def test_yylex_string_pct_I_extra_space
2517
- assert_lex3("%I[ s1 s2\ns3 ]",
2705
+ def test_yylex_string_pct_i_extra_space
2706
+ assert_lex3("%i[ s1 s2\ns3 ]",
2518
2707
  nil,
2519
- :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2708
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2520
2709
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2521
2710
  :tSPACE, nil, EXPR_BEG,
2522
2711
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2523
2712
  :tSPACE, nil, EXPR_BEG,
2524
2713
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2525
2714
  :tSPACE, nil, EXPR_BEG,
2526
- :tSTRING_END, nil, EXPR_END)
2715
+ :tSTRING_END, nil, EXPR_LIT)
2527
2716
  end
2528
2717
 
2529
- def test_yylex_string_pct_q
2530
- assert_lex3("%q[s1 s2]",
2718
+ def test_yylex_string_pct_other
2719
+ assert_lex3("%%blah%",
2531
2720
  nil,
2532
- :tSTRING_BEG, "%q[", EXPR_BEG,
2533
- :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2534
- :tSTRING_END, "]", EXPR_END)
2721
+ :tSTRING_BEG, "%%", EXPR_BEG,
2722
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2723
+ :tSTRING_END, "%", EXPR_LIT)
2535
2724
  end
2536
2725
 
2537
- def test_yylex_string_pct_Q
2538
- assert_lex3("%Q[s1 s2]",
2726
+ def test_yylex_string_pct_q
2727
+ assert_lex3("%q[s1 s2]",
2539
2728
  nil,
2540
- :tSTRING_BEG, "%Q[", EXPR_BEG,
2729
+ :tSTRING_BEG, "%q[", EXPR_BEG,
2541
2730
  :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2542
- :tSTRING_END, "]", EXPR_END)
2731
+ :tSTRING_END, "]", EXPR_LIT)
2543
2732
  end
2544
2733
 
2545
2734
  def test_yylex_string_pct_s
@@ -2547,109 +2736,151 @@ class TestRubyLexer < Minitest::Test
2547
2736
  nil,
2548
2737
  :tSYMBEG, "%s[", EXPR_FNAME, # TODO: :tSYM_BEG ?
2549
2738
  :tSTRING_CONTENT, "s1 s2", EXPR_FNAME, # man... I don't like this
2550
- :tSTRING_END, "]", EXPR_END)
2739
+ :tSTRING_END, "]", EXPR_LIT)
2551
2740
  end
2552
2741
 
2553
- def test_yylex_string_pct_W
2554
- assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2555
- nil,
2556
- :tWORDS_BEG, "%W[", EXPR_BEG,
2557
- :tSTRING_CONTENT, "s1", EXPR_BEG,
2558
- :tSPACE, nil, EXPR_BEG,
2559
- :tSTRING_CONTENT, "s2", EXPR_BEG,
2560
- :tSPACE, nil, EXPR_BEG,
2561
- :tSTRING_CONTENT, "s3", EXPR_BEG,
2562
- :tSPACE, nil, EXPR_BEG,
2563
- :tSTRING_END, nil, EXPR_END)
2742
+ def test_yylex_string_pct_w
2743
+ refute_lex("%w[s1 s2 ",
2744
+ :tQWORDS_BEG, "%w[",
2745
+ :tSTRING_CONTENT, "s1",
2746
+ :tSPACE, nil,
2747
+ :tSTRING_CONTENT, "s2",
2748
+ :tSPACE, nil)
2564
2749
  end
2565
2750
 
2566
- def test_yylex_string_pct_W_bs_nl
2567
- assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2751
+ def test_yylex_string_pct_w_bs_nl
2752
+ assert_lex3("%w[s1 \\\ns2]",
2568
2753
  nil,
2569
- :tWORDS_BEG, "%W[", EXPR_BEG,
2754
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2570
2755
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2571
2756
  :tSPACE, nil, EXPR_BEG,
2572
2757
  :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2573
2758
  :tSPACE, nil, EXPR_BEG,
2574
- :tSTRING_END, nil, EXPR_END)
2759
+ :tSTRING_END, nil, EXPR_LIT)
2760
+ end
2761
+
2762
+ def test_yylex_string_pct_w_bs_sp
2763
+ assert_lex3("%w[s\\ 1 s\\ 2]",
2764
+ nil,
2765
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2766
+ :tSTRING_CONTENT, "s 1", EXPR_BEG,
2767
+ :tSPACE, nil, EXPR_BEG,
2768
+ :tSTRING_CONTENT, "s 2", EXPR_BEG,
2769
+ :tSPACE, nil, EXPR_BEG,
2770
+ :tSTRING_END, nil, EXPR_LIT)
2771
+ end
2772
+
2773
+ def test_yylex_string_single
2774
+ assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
2775
+ end
2776
+
2777
+ def test_yylex_string_single_escape_chars
2778
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
2779
+ end
2780
+
2781
+ def test_yylex_string_single_escape_quote_and_backslash
2782
+ assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz",
2783
+ EXPR_LIT)
2784
+ end
2785
+
2786
+ def test_yylex_string_single_escaped_quote
2787
+ assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
2788
+ end
2789
+
2790
+ def test_yylex_string_single_nl
2791
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
2792
+ end
2793
+
2794
+ def test_yylex_string_utf8_complex
2795
+ chr = [0x3024].pack("U")
2796
+
2797
+ assert_lex3('"#@a\u{3024}"',
2798
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2799
+ :tSTRING_BEG, '"', EXPR_BEG,
2800
+ :tSTRING_DVAR, nil, EXPR_BEG,
2801
+ :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2802
+ :tSTRING_END, '"', EXPR_LIT)
2575
2803
  end
2576
2804
 
2577
- def test_yylex_string_pct_angle
2578
- assert_lex3("%<blah>",
2579
- nil,
2580
- :tSTRING_BEG, "%>", EXPR_BEG,
2581
- :tSTRING_CONTENT, "blah", EXPR_BEG,
2582
- :tSTRING_END, ">", EXPR_END)
2583
- end
2805
+ def test_yylex_string_utf8_complex_missing_hex
2806
+ chr = [0x302].pack("U")
2807
+ str = "#{chr}zzz"
2808
+
2809
+ refute_lex('"#@a\u302zzz"',
2810
+ :tSTRING_BEG, '"',
2811
+ :tSTRING_DVAR, nil,
2812
+ :tSTRING_CONTENT, "@a"+str,
2813
+ :tSTRING_END, '"')
2814
+
2815
+ chr = [0x30].pack("U")
2816
+ str = "#{chr}zzz"
2817
+
2818
+ refute_lex('"#@a\u30zzz"',
2819
+ :tSTRING_BEG, '"',
2820
+ :tSTRING_DVAR, nil,
2821
+ :tSTRING_CONTENT, "@a"+str,
2822
+ :tSTRING_END, '"')
2823
+
2824
+ chr = [0x3].pack("U")
2825
+ str = "#{chr}zzz"
2584
2826
 
2585
- def test_yylex_string_pct_other
2586
- assert_lex3("%%blah%",
2587
- nil,
2588
- :tSTRING_BEG, "%%", EXPR_BEG,
2589
- :tSTRING_CONTENT, "blah", EXPR_BEG,
2590
- :tSTRING_END, "%", EXPR_END)
2827
+ refute_lex('"#@a\u3zzz"',
2828
+ :tSTRING_BEG, '"',
2829
+ :tSTRING_DVAR, nil,
2830
+ :tSTRING_CONTENT, "@a"+str,
2831
+ :tSTRING_END, '"')
2591
2832
  end
2592
2833
 
2593
- def test_yylex_string_pct_w
2594
- refute_lex("%w[s1 s2 ",
2595
- :tQWORDS_BEG, "%w[",
2596
- :tSTRING_CONTENT, "s1",
2597
- :tSPACE, nil,
2598
- :tSTRING_CONTENT, "s2",
2599
- :tSPACE, nil)
2600
- end
2834
+ def test_yylex_string_utf8_complex_trailing_hex
2835
+ chr = [0x3024].pack("U")
2836
+ str = "#{chr}abz"
2601
2837
 
2602
- def test_yylex_string_pct_w_bs_nl
2603
- assert_lex3("%w[s1 \\\ns2]",
2604
- nil,
2605
- :tQWORDS_BEG, "%w[", EXPR_BEG,
2606
- :tSTRING_CONTENT, "s1", EXPR_BEG,
2607
- :tSPACE, nil, EXPR_BEG,
2608
- :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2609
- :tSPACE, nil, EXPR_BEG,
2610
- :tSTRING_END, nil, EXPR_END)
2838
+ assert_lex3('"#@a\u3024abz"',
2839
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2840
+ :tSTRING_BEG, '"', EXPR_BEG,
2841
+ :tSTRING_DVAR, nil, EXPR_BEG,
2842
+ :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2843
+ :tSTRING_END, '"', EXPR_LIT)
2611
2844
  end
2612
2845
 
2613
- def test_yylex_string_pct_w_bs_sp
2614
- assert_lex3("%w[s\\ 1 s\\ 2]",
2615
- nil,
2616
- :tQWORDS_BEG, "%w[", EXPR_BEG,
2617
- :tSTRING_CONTENT, "s 1", EXPR_BEG,
2618
- :tSPACE, nil, EXPR_BEG,
2619
- :tSTRING_CONTENT, "s 2", EXPR_BEG,
2620
- :tSPACE, nil, EXPR_BEG,
2621
- :tSTRING_END, nil, EXPR_END)
2846
+ def test_yylex_string_utf8_missing_hex
2847
+ refute_lex('"\u3zzz"')
2848
+ refute_lex('"\u30zzz"')
2849
+ refute_lex('"\u302zzz"')
2622
2850
  end
2623
2851
 
2624
- def test_yylex_string_single
2625
- assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
2626
- end
2852
+ def test_yylex_string_utf8_simple
2853
+ chr = [0x3024].pack("U")
2627
2854
 
2628
- def test_yylex_string_single_escape_chars
2629
- assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
2855
+ assert_lex3('"\u{3024}"',
2856
+ s(:str, chr),
2857
+ :tSTRING, chr, EXPR_END)
2630
2858
  end
2631
2859
 
2632
- def test_yylex_string_single_nl
2633
- assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
2634
- end
2860
+ def test_yylex_string_utf8_trailing_hex
2861
+ chr = [0x3024].pack("U")
2862
+ str = "#{chr}abz"
2635
2863
 
2636
- def test_yylex_string_single_escaped_quote
2637
- assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
2864
+ assert_lex3('"\u3024abz"',
2865
+ s(:str, str),
2866
+ :tSTRING, str, EXPR_END)
2638
2867
  end
2639
2868
 
2640
- def test_yylex_symbol
2641
- assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_END)
2869
+ def test_yylex_sym_quoted
2870
+ assert_lex(":'a'",
2871
+ s(:lit, :a),
2872
+
2873
+ :tSYMBOL, "a", EXPR_LIT, 0, 0)
2642
2874
  end
2643
2875
 
2644
- def test_yylex_symbol_zero_byte
2645
- assert_lex(":\"symbol\0\"", nil,
2646
- :tSYMBOL, "symbol\0", EXPR_END)
2876
+ def test_yylex_symbol
2877
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_LIT)
2647
2878
  end
2648
2879
 
2649
2880
  def test_yylex_symbol_double
2650
2881
  assert_lex3(":\"symbol\"",
2651
2882
  nil,
2652
- :tSYMBOL, "symbol", EXPR_END)
2883
+ :tSYMBOL, "symbol", EXPR_LIT)
2653
2884
  end
2654
2885
 
2655
2886
  def test_yylex_symbol_double_interp
@@ -2659,29 +2890,30 @@ class TestRubyLexer < Minitest::Test
2659
2890
  :tSTRING_CONTENT, "symbol", EXPR_FNAME,
2660
2891
  :tSTRING_DBEG, nil, EXPR_FNAME,
2661
2892
  :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
2662
- :tSTRING_END, "\"", EXPR_END)
2893
+ :tSTRING_END, "\"", EXPR_LIT)
2663
2894
  end
2664
2895
 
2665
2896
  def test_yylex_symbol_single
2666
2897
  assert_lex3(":'symbol'",
2667
2898
  nil,
2668
- :tSYMBOL, "symbol", EXPR_END)
2899
+ :tSYMBOL, "symbol", EXPR_LIT)
2669
2900
  end
2670
2901
 
2671
- def test_yylex_symbol_single_noninterp
2672
- assert_lex3(':\'symbol#{1+1}\'',
2902
+ def test_yylex_symbol_single_escape_chars
2903
+ assert_lex3(":'s\\tri\\ng'",
2673
2904
  nil,
2674
- :tSYMBOL, 'symbol#{1+1}', EXPR_END)
2905
+ :tSYMBOL, "s\\tri\\ng", EXPR_LIT)
2675
2906
  end
2676
2907
 
2677
- def test_yylex_symbol_single_escape_chars
2678
- assert_lex3(":'s\\tri\\ng'",
2908
+ def test_yylex_symbol_single_noninterp
2909
+ assert_lex3(':\'symbol#{1+1}\'',
2679
2910
  nil,
2680
- :tSYMBOL, "s\\tri\\ng", EXPR_END)
2911
+ :tSYMBOL, 'symbol#{1+1}', EXPR_LIT)
2681
2912
  end
2682
2913
 
2683
- def test_yylex_string_single_escape_quote_and_backslash
2684
- assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz", EXPR_END)
2914
+ def test_yylex_symbol_zero_byte
2915
+ assert_lex(":\"symbol\0\"", nil,
2916
+ :tSYMBOL, "symbol\0", EXPR_LIT)
2685
2917
  end
2686
2918
 
2687
2919
  def test_yylex_ternary1
@@ -2703,7 +2935,7 @@ class TestRubyLexer < Minitest::Test
2703
2935
 
2704
2936
  assert_lex3("42 ?",
2705
2937
  nil,
2706
- :tINTEGER, 42, EXPR_NUM,
2938
+ :tINTEGER, 42, EXPR_NUM,
2707
2939
  :tEH, "?", EXPR_BEG)
2708
2940
  end
2709
2941
 
@@ -2746,14 +2978,14 @@ class TestRubyLexer < Minitest::Test
2746
2978
  nil,
2747
2979
  :kDEF, "def", EXPR_FNAME,
2748
2980
  :tIDENTIFIER, "initialize", EXPR_ENDFN,
2749
- :tLPAREN2, "(", EXPR_PAR,
2981
+ :tLPAREN2, "(", EXPR_PAR,
2750
2982
  :tIDENTIFIER, "u", EXPR_ARG,
2751
2983
  :tEQL, "=", EXPR_BEG,
2752
- :tFLOAT, 0.0, EXPR_NUM,
2753
- :tCOMMA, ",", EXPR_PAR,
2984
+ :tFLOAT, 0.0, EXPR_NUM,
2985
+ :tCOMMA, ",", EXPR_PAR,
2754
2986
  :tIDENTIFIER, "s", EXPR_ARG,
2755
2987
  :tEQL, "=", EXPR_BEG,
2756
- :tFLOAT, 0.0, EXPR_NUM)
2988
+ :tFLOAT, 0.0, EXPR_NUM)
2757
2989
  end
2758
2990
 
2759
2991
  def test_zbug_id_equals
@@ -2761,7 +2993,7 @@ class TestRubyLexer < Minitest::Test
2761
2993
  nil,
2762
2994
  :tIDENTIFIER, "a", EXPR_CMDARG,
2763
2995
  :tEQL, "=", EXPR_BEG,
2764
- :tFLOAT, 0.0, EXPR_NUM)
2996
+ :tFLOAT, 0.0, EXPR_NUM)
2765
2997
  end
2766
2998
 
2767
2999
  def test_zbug_no_spaces_in_decl
@@ -2769,190 +3001,13 @@ class TestRubyLexer < Minitest::Test
2769
3001
  nil,
2770
3002
  :kDEF, "def", EXPR_FNAME,
2771
3003
  :tIDENTIFIER, "initialize", EXPR_ENDFN,
2772
- :tLPAREN2, "(", EXPR_PAR,
3004
+ :tLPAREN2, "(", EXPR_PAR,
2773
3005
  :tIDENTIFIER, "u", EXPR_ARG,
2774
3006
  :tEQL, "=", EXPR_BEG,
2775
- :tFLOAT, 0.0, EXPR_NUM,
2776
- :tCOMMA, ",", EXPR_PAR,
3007
+ :tFLOAT, 0.0, EXPR_NUM,
3008
+ :tCOMMA, ",", EXPR_PAR,
2777
3009
  :tIDENTIFIER, "s", EXPR_ARG,
2778
3010
  :tEQL, "=", EXPR_BEG,
2779
- :tFLOAT, 0.0, EXPR_NUM)
2780
- end
2781
-
2782
- def test_pct_w_backslashes
2783
- ["\t", "\n", "\r", "\v", "\f"].each do |char|
2784
- next if !RubyLexer::HAS_ENC and char == "\v"
2785
-
2786
- assert_lex("%w[foo#{char}bar]",
2787
- s(:array, s(:str, "foo"), s(:str, "bar")),
2788
-
2789
- :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
2790
- :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
2791
- :tSPACE, nil, EXPR_BEG, 0, 0,
2792
- :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
2793
- :tSPACE, nil, EXPR_BEG, 0, 0,
2794
- :tSTRING_END, nil, EXPR_END, 0, 0)
2795
- end
2796
- end
2797
-
2798
- def test_yylex_sym_quoted
2799
- assert_lex(":'a'",
2800
- s(:lit, :a),
2801
-
2802
- :tSYMBOL, "a", EXPR_END, 0, 0)
2803
- end
2804
-
2805
- def test_yylex_hash_colon
2806
- assert_lex("{a:1}",
2807
- s(:hash, s(:lit, :a), s(:lit, 1)),
2808
-
2809
- :tLBRACE, "{", EXPR_PAR, 0, 1,
2810
- :tLABEL, "a", EXPR_LAB, 0, 1,
2811
- :tINTEGER, 1, EXPR_NUM, 0, 1,
2812
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
2813
- end
2814
-
2815
- def test_yylex_hash_colon_quoted_22
2816
- setup_lexer_class RubyParser::V22
2817
-
2818
- assert_lex("{'a':1}",
2819
- s(:hash, s(:lit, :a), s(:lit, 1)),
2820
-
2821
- :tLBRACE, "{", EXPR_PAR, 0, 1,
2822
- :tLABEL, "a", EXPR_LAB, 0, 1,
2823
- :tINTEGER, 1, EXPR_NUM, 0, 1,
2824
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
2825
- end
2826
-
2827
- def test_yylex_hash_colon_quoted_symbol_22
2828
- setup_lexer_class RubyParser::V22
2829
-
2830
- assert_lex("{'abc': :b}",
2831
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2832
-
2833
- :tLBRACE, "{", EXPR_PAR, 0, 1,
2834
- :tLABEL, "abc", EXPR_LAB, 0, 1,
2835
- :tSYMBOL, "b", EXPR_END, 0, 1,
2836
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
2837
- end
2838
-
2839
- def test_yylex_hash_colon_double_quoted_symbol_22
2840
- setup_lexer_class RubyParser::V22
2841
-
2842
- assert_lex('{"abc": :b}',
2843
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2844
-
2845
- :tLBRACE, "{", EXPR_PAR, 0, 1,
2846
- :tLABEL, "abc", EXPR_LAB, 0, 1,
2847
- :tSYMBOL, "b", EXPR_END, 0, 1,
2848
- :tRCURLY, "}", EXPR_ENDARG, 0, 0)
2849
- end
2850
-
2851
- def test_yylex_required_kwarg_no_value_22
2852
- setup_lexer_class RubyParser::V22
2853
-
2854
- assert_lex3("def foo a:, b:\nend",
2855
- nil,
2856
- :kDEF, "def", EXPR_FNAME,
2857
- :tIDENTIFIER, "foo", EXPR_ENDFN,
2858
- :tLABEL, "a", EXPR_LAB,
2859
- :tCOMMA, ",", EXPR_PAR,
2860
- :tLABEL, "b", EXPR_LAB,
2861
- :kEND, "end", EXPR_END)
2862
- end
2863
-
2864
- def test_yylex_hash_colon_double_quoted_with_escapes
2865
- setup_lexer_class RubyParser::V22
2866
-
2867
- assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
2868
- nil,
2869
-
2870
- :tLBRACE, "{", EXPR_PAR,
2871
- :tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
2872
- :tINTEGER, 1, EXPR_NUM,
2873
- :tRCURLY, "}", EXPR_ENDARG)
2874
- end
2875
-
2876
- def test_yylex_hash_colon_quoted_with_escapes
2877
- setup_lexer_class RubyParser::V22
2878
-
2879
- assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
2880
- nil,
2881
-
2882
- :tLBRACE, "{", EXPR_PAR,
2883
- :tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
2884
- :tINTEGER, 1, EXPR_NUM,
2885
- :tRCURLY, "}", EXPR_ENDARG)
2886
- end
2887
-
2888
- def test_ruby21_rational_literal
2889
- setup_lexer_class RubyParser::V21
2890
-
2891
- assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
2892
- assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
2893
- assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
2894
- assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
2895
- assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
2896
- assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
2897
- assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
2898
- assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
2899
- assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
2900
- assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
2901
-
2902
- assert_lex3("-10r", nil,
2903
- :tUMINUS_NUM, "-", EXPR_BEG,
2904
- :tRATIONAL, Rational(10), EXPR_NUM)
2905
- end
2906
-
2907
- def test_ruby21_imaginary_literal
2908
- setup_lexer_class RubyParser::V21
2909
-
2910
- assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
2911
- assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
2912
- assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
2913
- assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
2914
- assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
2915
- assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
2916
- assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
2917
- assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
2918
- assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
2919
- assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
2920
-
2921
- assert_lex3("-10i", nil,
2922
- :tUMINUS_NUM, "-", EXPR_BEG,
2923
- :tIMAGINARY, Complex(0, 10), EXPR_NUM)
2924
- end
2925
-
2926
- def test_ruby21_rational_imaginary_literal
2927
- setup_lexer_class RubyParser::V21
2928
-
2929
- assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
2930
- assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
2931
- assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
2932
- assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
2933
- assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
2934
- assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
2935
- assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
2936
- assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
2937
- assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
2938
- assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
2939
-
2940
- assert_lex3("-10ri", nil,
2941
- :tUMINUS_NUM, "-", EXPR_BEG,
2942
- :tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
2943
- end
2944
-
2945
- def test_ruby21_imaginary_literal_with_succeeding_keyword
2946
- setup_lexer_class RubyParser::V21
2947
-
2948
- # 2/4 scenarios are syntax errors on all tested versions so I
2949
- # deleted them.
2950
-
2951
- assert_lex3("1if", nil,
2952
- :tINTEGER, 1, EXPR_NUM,
2953
- :kIF_MOD, "if", EXPR_PAR)
2954
- assert_lex3("1.0if", nil,
2955
- :tFLOAT, 1.0, EXPR_NUM,
2956
- :kIF_MOD, "if", EXPR_PAR)
3011
+ :tFLOAT, 0.0, EXPR_NUM)
2957
3012
  end
2958
3013
  end