ruby_parser 3.13.0 → 3.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.autotest +18 -29
  5. data/History.rdoc +123 -0
  6. data/Manifest.txt +2 -0
  7. data/README.rdoc +3 -3
  8. data/Rakefile +41 -24
  9. data/bin/ruby_parse +3 -1
  10. data/bin/ruby_parse_extract_error +18 -35
  11. data/compare/normalize.rb +43 -3
  12. data/debugging.md +39 -0
  13. data/lib/rp_extensions.rb +1 -1
  14. data/lib/ruby20_parser.rb +3654 -3466
  15. data/lib/ruby20_parser.y +504 -327
  16. data/lib/ruby21_parser.rb +3643 -3455
  17. data/lib/ruby21_parser.y +512 -334
  18. data/lib/ruby22_parser.rb +3669 -3492
  19. data/lib/ruby22_parser.y +513 -335
  20. data/lib/ruby23_parser.rb +3692 -3499
  21. data/lib/ruby23_parser.y +513 -335
  22. data/lib/ruby24_parser.rb +3685 -3463
  23. data/lib/ruby24_parser.y +517 -331
  24. data/lib/ruby25_parser.rb +3685 -3462
  25. data/lib/ruby25_parser.y +517 -331
  26. data/lib/ruby26_parser.rb +3696 -3471
  27. data/lib/ruby26_parser.y +523 -335
  28. data/lib/ruby27_parser.rb +7224 -0
  29. data/lib/ruby27_parser.y +2657 -0
  30. data/lib/ruby_lexer.rb +611 -495
  31. data/lib/ruby_lexer.rex +27 -28
  32. data/lib/ruby_lexer.rex.rb +71 -31
  33. data/lib/ruby_parser.rb +31 -27
  34. data/lib/ruby_parser.yy +529 -336
  35. data/lib/ruby_parser_extras.rb +720 -449
  36. data/test/test_ruby_lexer.rb +1560 -1412
  37. data/test/test_ruby_parser.rb +2611 -1912
  38. data/test/test_ruby_parser_extras.rb +39 -4
  39. data/tools/munge.rb +12 -6
  40. data/tools/ripper.rb +19 -3
  41. metadata +25 -18
  42. metadata.gz.sig +4 -1
data/test/test_ruby_lexer.rb CHANGED
@@ -1,29 +1,24 @@
1
- # encoding: US-ASCII
2
-
3
1
  require "minitest/autorun"
4
2
  require "ruby_lexer"
5
3
  require "ruby_parser"
6
4
 
7
5
  class TestRubyLexer < Minitest::Test
6
+ include RubyLexer::State::Values
7
+
8
8
  attr_accessor :processor, :lex, :parser_class, :lex_state
9
9
 
10
- alias :lexer :lex # lets me copy/paste code from parser
11
- alias :lexer= :lex=
10
+ alias lexer lex # lets me copy/paste code from parser
11
+ alias lexer= lex=
12
12
 
13
13
  def setup
14
- self.lex_state = :expr_beg
14
+ self.lex_state = EXPR_BEG
15
15
  setup_lexer_class RubyParser.latest.class
16
16
  end
17
17
 
18
18
  def setup_lexer input, exp_sexp = nil
19
19
  setup_new_parser
20
20
  lex.ss = RPStringScanner.new(input)
21
- lex.lex_state = self.lex_state
22
- end
23
-
24
- def setup_new_parser
25
- self.processor = parser_class.new
26
- self.lex = processor.lexer
21
+ lex.lex_state = lex_state
27
22
  end
28
23
 
29
24
  def setup_lexer_class parser_class
@@ -32,11 +27,16 @@ class TestRubyLexer < Minitest::Test
32
27
  setup_lexer "blah blah"
33
28
  end
34
29
 
35
- def assert_lex input, exp_sexp, *args, &b
30
+ def setup_new_parser
31
+ self.processor = parser_class.new
32
+ self.lex = processor.lexer
33
+ end
34
+
35
+ def assert_lex input, exp_sexp, *args
36
36
  setup_lexer input
37
37
  assert_parse input, exp_sexp if exp_sexp
38
38
 
39
- b.call if b
39
+ yield if block_given?
40
40
 
41
41
  args.each_slice(5) do |token, value, state, paren, brace|
42
42
  assert_next_lexeme token, value, state, paren, brace
@@ -51,48 +51,41 @@ class TestRubyLexer < Minitest::Test
51
51
  assert_lex(input, exp_sexp, *args, &block)
52
52
  end
53
53
 
54
- def refute_lex input, *args # TODO: re-sort
55
- args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
56
-
57
- assert_raises RubyParser::SyntaxError do
58
- assert_lex(input, nil, *args)
59
- end
60
- end
61
-
62
- def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
54
+ def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
63
55
  assert_lex3("def #{name} ",
64
56
  nil,
65
57
 
66
- :kDEF, "def", :expr_fname,
58
+ :kDEF, "def", EXPR_FNAME,
67
59
  type, name, end_state)
68
60
  end
69
61
 
70
62
  def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
71
63
  adv = @lex.next_token
72
64
 
73
- assert adv, "no more tokens"
65
+ assert adv, "no more tokens, expecting: %p %p %p %p %p" % [token, value, state, paren, brace]
74
66
 
75
67
  act_token, act_value = adv
76
68
 
77
69
  msg = message {
78
- act = [act_token, act_value, @lex.lex_state,
79
- @lex.paren_nest, @lex.brace_nest]
70
+ act = [act_token, act_value, @lex.lex_state, @lex.paren_nest, @lex.brace_nest]
80
71
  exp = [token, value, state, paren, brace]
81
72
  "#{exp.inspect} vs #{act.inspect}"
82
73
  }
83
74
 
84
75
  act_value = act_value.first if Array === act_value
85
76
 
86
- assert_equal token, act_token, msg
77
+ assert_equal token, act_token, msg
87
78
  case value
88
79
  when Float then
89
80
  assert_in_epsilon value, act_value, 0.001, msg
90
81
  when NilClass then
91
82
  assert_nil act_value, msg
83
+ when String then
84
+ assert_equal value, act_value.b.force_encoding(value.encoding), msg
92
85
  else
93
- assert_equal value, act_value, msg
86
+ assert_equal value, act_value, msg
94
87
  end
95
- assert_equal state, @lex.lex_state, msg if state
88
+ assert_match state, @lex.lex_state, msg if state
96
89
  assert_equal paren, @lex.paren_nest, msg if paren
97
90
  assert_equal brace, @lex.brace_nest, msg if brace
98
91
  end
@@ -102,8 +95,9 @@ class TestRubyLexer < Minitest::Test
102
95
  end
103
96
 
104
97
  def assert_read_escape expected, input
105
- @lex.ss.string = input
106
- assert_equal expected, @lex.read_escape, input
98
+ @lex.ss.string = input.dup
99
+ enc = expected.encoding
100
+ assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
107
101
  end
108
102
 
109
103
  def assert_read_escape_bad input # TODO: rename refute_read_escape
@@ -113,6 +107,20 @@ class TestRubyLexer < Minitest::Test
113
107
  end
114
108
  end
115
109
 
110
+ def refute_lex input, *args # TODO: re-sort
111
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
112
+
113
+ assert_raises RubyParser::SyntaxError do
114
+ assert_lex(input, nil, *args)
115
+ end
116
+ end
117
+
118
+ def refute_lex5 input, *args
119
+ assert_raises RubyParser::SyntaxError do
120
+ assert_lex(input, *args)
121
+ end
122
+ end
123
+
116
124
  def refute_lexeme
117
125
  x = y = @lex.next_token
118
126
 
@@ -132,15 +140,15 @@ class TestRubyLexer < Minitest::Test
132
140
  lexer.cmdarg.push false
133
141
 
134
142
  lexer.lex_strterm = nil
135
- lexer.lex_state = :expr_beg
143
+ lexer.lex_state = EXPR_BEG
136
144
 
137
145
  yield
138
146
 
139
- lexer.lex_state = :expr_endarg
140
- assert_next_lexeme :tSTRING_DEND, "}", :expr_endarg, 0
147
+ lexer.lex_state = EXPR_ENDARG
148
+ assert_next_lexeme :tSTRING_DEND, "}", EXPR_END|EXPR_ENDARG, 0
141
149
 
142
150
  lexer.lex_strterm = lex_strterm
143
- lexer.lex_state = :expr_beg
151
+ lexer.lex_state = EXPR_BEG
144
152
  lexer.string_nest = string_nest
145
153
  lexer.brace_nest = brace_nest
146
154
 
@@ -156,9 +164,20 @@ class TestRubyLexer < Minitest::Test
156
164
  assert_nil @lex.next_token
157
165
  end
158
166
 
159
- def test_unicode_ident
160
- s = "@\u1088\u1077\u1093\u1072"
161
- assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
167
+ def test_pct_w_backslashes
168
+ ["\t", "\n", "\r", "\v", "\f"].each do |char|
169
+ next if !RubyLexer::HAS_ENC and char == "\v"
170
+
171
+ assert_lex("%w[foo#{char}bar]",
172
+ s(:array, s(:str, "foo"), s(:str, "bar")),
173
+
174
+ :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
175
+ :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
176
+ :tSPACE, nil, EXPR_BEG, 0, 0,
177
+ :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
178
+ :tSPACE, nil, EXPR_BEG, 0, 0,
179
+ :tSTRING_END, nil, EXPR_LIT, 0, 0)
180
+ end
162
181
  end
163
182
 
164
183
  def test_read_escape
@@ -214,12 +233,96 @@ class TestRubyLexer < Minitest::Test
214
233
  assert_read_escape "\230", 'M-\cx'
215
234
  end
216
235
 
236
+ def test_ruby21_imaginary_literal
237
+ setup_lexer_class RubyParser::V21
238
+
239
+ assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
240
+ assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
241
+ assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
242
+ assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
243
+ assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
244
+ assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
245
+ assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
246
+ assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
247
+ assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
248
+ assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
249
+
250
+ c010 = Complex(0, 10)
251
+ assert_lex3("-10i", nil,
252
+ :tUMINUS_NUM, "-", EXPR_BEG,
253
+ :tIMAGINARY, c010, EXPR_NUM)
254
+ end
255
+
256
+ def test_ruby21_imaginary_literal_with_succeeding_keyword
257
+ setup_lexer_class RubyParser::V21
258
+
259
+ # 2/4 scenarios are syntax errors on all tested versions so I
260
+ # deleted them.
261
+
262
+ assert_lex3("1if", nil,
263
+ :tINTEGER, 1, EXPR_NUM,
264
+ :kIF_MOD, "if", EXPR_PAR)
265
+ assert_lex3("1.0if", nil,
266
+ :tFLOAT, 1.0, EXPR_NUM,
267
+ :kIF_MOD, "if", EXPR_PAR)
268
+ end
269
+
270
+ def test_ruby21_rational_imaginary_literal
271
+ setup_lexer_class RubyParser::V21
272
+
273
+ assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
274
+ assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
275
+ assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
276
+ assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
277
+ assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
278
+ assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
279
+ assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
280
+ assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
281
+ assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
282
+ assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
283
+
284
+ assert_lex3("-10ri", nil,
285
+ :tUMINUS_NUM, "-", EXPR_BEG,
286
+ :tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
287
+ end
288
+
289
+ def test_ruby21_rational_literal
290
+ setup_lexer_class RubyParser::V21
291
+
292
+ assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
293
+ assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
294
+ assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
295
+ assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
296
+ assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
297
+ assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
298
+ assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
299
+ assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
300
+ assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
301
+ assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
302
+
303
+ r10 = Rational(10)
304
+ assert_lex3("-10r", nil,
305
+ :tUMINUS_NUM, "-", EXPR_BEG,
306
+ :tRATIONAL, r10, EXPR_NUM)
307
+ end
308
+
309
+ def test_unicode_ident
310
+ s = "@\u1088\u1077\u1093\u1072"
311
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
312
+ end
313
+
314
+ def test_why_does_ruby_hate_me?
315
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
316
+ nil,
317
+ :tSTRING, %W[ Nl% \u0000 \u0000 A \u0000 999 ].join, EXPR_END)
318
+ end
319
+
217
320
  def test_yylex_ambiguous_uminus
218
321
  assert_lex3("m -3",
219
322
  nil,
220
- :tIDENTIFIER, "m", :expr_cmdarg,
221
- :tUMINUS_NUM, "-", :expr_beg,
222
- :tINTEGER, 3, :expr_end)
323
+ :tIDENTIFIER, "m", EXPR_CMDARG,
324
+ :tUMINUS_NUM, "-", EXPR_BEG,
325
+ :tINTEGER, 3, EXPR_NUM)
223
326
 
224
327
  # TODO: verify warning
225
328
  end
@@ -227,36 +330,45 @@ class TestRubyLexer < Minitest::Test
227
330
  def test_yylex_ambiguous_uplus
228
331
  assert_lex3("m +3",
229
332
  nil,
230
- :tIDENTIFIER, "m", :expr_cmdarg,
231
- :tINTEGER, 3, :expr_end)
333
+ :tIDENTIFIER, "m", EXPR_CMDARG,
334
+ :tINTEGER, 3, EXPR_NUM)
232
335
 
233
336
  # TODO: verify warning
234
337
  end
235
338
 
236
339
  def test_yylex_and
237
- assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
340
+ assert_lex3("&", nil, :tAMPER, "&", EXPR_BEG)
238
341
  end
239
342
 
240
343
  def test_yylex_and2
241
- assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
344
+ assert_lex3("&&", nil, :tANDOP, "&&", EXPR_BEG)
242
345
  end
243
346
 
244
347
  def test_yylex_and2_equals
245
- assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
348
+ assert_lex3("&&=", nil, :tOP_ASGN, "&&", EXPR_BEG)
349
+ end
350
+
351
+ def test_yylex_and_arg
352
+ self.lex_state = EXPR_ARG
353
+
354
+ assert_lex3(" &y",
355
+ nil,
356
+ :tAMPER, "&", EXPR_BEG,
357
+ :tIDENTIFIER, "y", EXPR_ARG)
246
358
  end
247
359
 
248
360
  def test_yylex_and_dot
249
361
  setup_lexer_class RubyParser::V23
250
362
 
251
- assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
363
+ assert_lex3("&.", nil, :tLONELY, "&.", EXPR_DOT)
252
364
  end
253
365
 
254
366
  def test_yylex_and_dot_call
255
367
  setup_lexer_class RubyParser::V23
256
368
 
257
369
  assert_lex3("x&.y", nil,
258
- :tIDENTIFIER, "x", :expr_cmdarg,
259
- :tLONELY, "&.", :expr_dot,
370
+ :tIDENTIFIER, "x", EXPR_CMDARG,
371
+ :tLONELY, "&.", EXPR_DOT,
260
372
  :tIDENTIFIER, "y")
261
373
  end
262
374
 
@@ -264,32 +376,23 @@ class TestRubyLexer < Minitest::Test
264
376
  setup_lexer_class Ruby23Parser
265
377
 
266
378
  assert_lex3("x\n&.y", nil,
267
- :tIDENTIFIER, "x", :expr_cmdarg,
268
- :tLONELY, "&.", :expr_dot,
379
+ :tIDENTIFIER, "x", EXPR_CMDARG,
380
+ :tLONELY, "&.", EXPR_DOT,
269
381
  :tIDENTIFIER, "y")
270
382
  end
271
383
 
272
- def test_yylex_and_arg
273
- self.lex_state = :expr_arg
274
-
275
- assert_lex3(" &y",
276
- nil,
277
- :tAMPER, "&", :expr_beg,
278
- :tIDENTIFIER, "y", :expr_arg)
279
- end
280
-
281
384
  def test_yylex_and_equals
282
- assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
385
+ assert_lex3("&=", nil, :tOP_ASGN, "&", EXPR_BEG)
283
386
  end
284
387
 
285
388
  def test_yylex_and_expr
286
- self.lex_state = :expr_arg
389
+ self.lex_state = EXPR_ARG
287
390
 
288
391
  assert_lex3("x & y",
289
392
  nil,
290
- :tIDENTIFIER, "x", :expr_cmdarg,
291
- :tAMPER2, "&", :expr_beg,
292
- :tIDENTIFIER, "y", :expr_arg)
393
+ :tIDENTIFIER, "x", EXPR_CMDARG,
394
+ :tAMPER2, "&", EXPR_BEG,
395
+ :tIDENTIFIER, "y", EXPR_ARG)
293
396
  end
294
397
 
295
398
  def test_yylex_and_meth
@@ -297,338 +400,26 @@ class TestRubyLexer < Minitest::Test
297
400
  end
298
401
 
299
402
  def test_yylex_assoc
300
- assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
301
- end
302
-
303
- def test_yylex_label__20
304
- setup_lexer_class RubyParser::V20
305
-
306
- assert_lex3("{a:",
307
- nil,
308
- :tLBRACE, "{", :expr_beg,
309
- :tLABEL, "a", :expr_labeled)
310
- end
311
-
312
- def test_yylex_label_in_params__20
313
- setup_lexer_class RubyParser::V20
314
-
315
- assert_lex3("foo(a:",
316
- nil,
317
- :tIDENTIFIER, "foo", :expr_cmdarg,
318
- :tLPAREN2, "(", :expr_beg,
319
- :tLABEL, "a", :expr_labeled)
320
- end
321
-
322
- def test_yylex_paren_string_parens_interpolated
323
- setup_lexer('%((#{b}#{d}))',
324
- s(:dstr,
325
- "(",
326
- s(:evstr, s(:call, nil, :b)),
327
- s(:evstr, s(:call, nil, :d)),
328
- s(:str, ")")))
329
-
330
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
331
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
332
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
333
-
334
- emulate_string_interpolation do
335
- assert_next_lexeme :tIDENTIFIER, "b", :expr_cmdarg, 0, 0
336
- end
337
-
338
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
339
-
340
- emulate_string_interpolation do
341
- assert_next_lexeme :tIDENTIFIER, "d", :expr_cmdarg, 0, 0
342
- end
343
-
344
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
345
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
346
-
347
- refute_lexeme
348
- end
349
-
350
- def test_yylex_paren_string_interpolated_regexp
351
- setup_lexer('%( #{(/abcd/)} )',
352
- s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
353
-
354
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
355
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
356
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
357
-
358
- emulate_string_interpolation do
359
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
360
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
361
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
362
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
363
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
364
- end
365
-
366
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
367
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
368
-
369
- refute_lexeme
370
- end
371
-
372
- def test_yylex_not_at_defn__20
373
- setup_lexer_class RubyParser::V20
374
-
375
- assert_lex("def +@; end",
376
- s(:defn, :+@, s(:args), s(:nil)),
377
-
378
- :kDEF, "def", :expr_fname, 0, 0,
379
- :tUPLUS, "+@", :expr_arg, 0, 0,
380
- :tSEMI, ";", :expr_beg, 0, 0,
381
- :kEND, "end", :expr_end, 0, 0)
382
-
383
- assert_lex("def !@; end",
384
- s(:defn, :"!@", s(:args), s(:nil)),
385
-
386
- :kDEF, "def", :expr_fname, 0, 0,
387
- :tUBANG, "!@", :expr_arg, 0, 0,
388
- :tSEMI, ";", :expr_beg, 0, 0,
389
- :kEND, "end", :expr_end, 0, 0)
390
- end
391
-
392
- def test_yylex_not_at_ivar
393
- assert_lex("!@ivar",
394
- s(:call, s(:ivar, :@ivar), :"!"),
395
-
396
- :tBANG, "!", :expr_beg, 0, 0,
397
- :tIVAR, "@ivar", :expr_end, 0, 0)
398
- end
399
-
400
- def test_yylex_number_times_ident_times_return_number
401
- assert_lex("1 * b * 3",
402
- s(:call,
403
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
404
- :*, s(:lit, 3)),
405
-
406
- :tINTEGER, 1, :expr_end, 0, 0,
407
- :tSTAR2, "*", :expr_beg, 0, 0,
408
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
409
- :tSTAR2, "*", :expr_beg, 0, 0,
410
- :tINTEGER, 3, :expr_end, 0, 0)
411
-
412
- assert_lex("1 * b *\n 3",
413
- s(:call,
414
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
415
- :*, s(:lit, 3)),
416
-
417
- :tINTEGER, 1, :expr_end, 0, 0,
418
- :tSTAR2, "*", :expr_beg, 0, 0,
419
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
420
- :tSTAR2, "*", :expr_beg, 0, 0,
421
- :tINTEGER, 3, :expr_end, 0, 0)
422
- end
423
-
424
- def test_yylex_paren_string_parens_interpolated_regexp
425
- setup_lexer('%((#{(/abcd/)}))',
426
- s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
427
-
428
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
429
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
430
-
431
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
432
-
433
- emulate_string_interpolation do
434
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
435
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
436
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
437
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
438
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
439
- end
440
-
441
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
442
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
443
-
444
- refute_lexeme
445
- end
446
-
447
- def test_yylex_method_parens_chevron
448
- assert_lex("a()<<1",
449
- s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
450
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
451
- :tLPAREN2, "(", :expr_beg, 1, 0,
452
- :tRPAREN, ")", :expr_endfn, 0, 0,
453
- :tLSHFT, "<<" , :expr_beg, 0, 0,
454
- :tINTEGER, 1, :expr_end, 0, 0)
455
- end
456
-
457
- def test_yylex_lambda_args__20
458
- setup_lexer_class RubyParser::V20
459
-
460
- assert_lex("-> (a) { }",
461
- s(:iter, s(:call, nil, :lambda),
462
- s(:args, :a)),
463
-
464
- :tLAMBDA, nil, :expr_endfn, 0, 0,
465
- :tLPAREN2, "(", :expr_beg, 1, 0,
466
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
467
- :tRPAREN, ")", :expr_endfn, 0, 0,
468
- :tLCURLY, "{", :expr_beg, 0, 1,
469
- :tRCURLY, "}", :expr_endarg, 0, 0)
470
- end
471
-
472
- def test_yylex_lambda_as_args_with_block__20
473
- setup_lexer_class RubyParser::V20
474
-
475
- assert_lex3("a -> do end do end",
476
- nil,
477
- :tIDENTIFIER, "a", :expr_cmdarg,
478
- :tLAMBDA, nil, :expr_endfn,
479
- :kDO, "do", :expr_beg,
480
- :kEND, "end", :expr_end,
481
- :kDO, "do", :expr_beg,
482
- :kEND, "end", :expr_end)
483
- end
484
-
485
- def test_yylex_lambda_args_opt__20
486
- setup_lexer_class RubyParser::V20
487
-
488
- assert_lex("-> (a=nil) { }",
489
- s(:iter, s(:call, nil, :lambda),
490
- s(:args, s(:lasgn, :a, s(:nil)))),
491
-
492
- :tLAMBDA, nil, :expr_endfn, 0, 0,
493
- :tLPAREN2, "(", :expr_beg, 1, 0,
494
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
495
- :tEQL, "=", :expr_beg, 1, 0,
496
- :kNIL, "nil", :expr_end, 1, 0,
497
- :tRPAREN, ")", :expr_endfn, 0, 0,
498
- :tLCURLY, "{", :expr_beg, 0, 1,
499
- :tRCURLY, "}", :expr_endarg, 0, 0)
500
- end
501
-
502
- def test_yylex_lambda_hash__20
503
- setup_lexer_class RubyParser::V20
504
-
505
- assert_lex("-> (a={}) { }",
506
- s(:iter, s(:call, nil, :lambda),
507
- s(:args, s(:lasgn, :a, s(:hash)))),
508
-
509
- :tLAMBDA, nil, :expr_endfn, 0, 0,
510
- :tLPAREN2, "(", :expr_beg, 1, 0,
511
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
512
- :tEQL, "=", :expr_beg, 1, 0,
513
- :tLBRACE, "{", :expr_beg, 1, 1,
514
- :tRCURLY, "}", :expr_endarg, 1, 0,
515
- :tRPAREN, ")", :expr_endfn, 0, 0,
516
- :tLCURLY, "{", :expr_beg, 0, 1,
517
- :tRCURLY, "}", :expr_endarg, 0, 0)
518
- end
519
-
520
- def test_yylex_iter_array_curly
521
- assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
522
- s(:iter,
523
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
524
- s(:args, :c, :d)),
525
-
526
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 0,
527
- :tSYMBOL, "a", :expr_end, 0, 0,
528
- :tCOMMA, ",", :expr_beg, 0, 0,
529
- :tLBRACK, "[", :expr_beg, 1, 0,
530
- :tSYMBOL, "b", :expr_end, 1, 0,
531
- :tRBRACK, "]", :expr_endarg, 0, 0,
532
- :tLBRACE_ARG, "{", :expr_beg, 0, 1,
533
- :tPIPE, "|", :expr_beg, 0, 1,
534
- :tIDENTIFIER, "c", :expr_arg, 0, 1,
535
- :tCOMMA, ",", :expr_beg, 0, 1,
536
- :tIDENTIFIER, "d", :expr_arg, 0, 1,
537
- :tPIPE, "|", :expr_beg, 0, 1,
538
- :tRCURLY, "}", :expr_endarg, 0, 0)
539
- end
540
-
541
- def test_yylex_const_call_same_name
542
- assert_lex("X = a { }; b { f :c }",
543
- s(:block,
544
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
545
- s(:iter,
546
- s(:call, nil, :b),
547
- 0,
548
- s(:call, nil, :f, s(:lit, :c)))),
549
-
550
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
551
- :tEQL, "=", :expr_beg, 0, 0,
552
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
553
- :tLCURLY, "{", :expr_beg, 0, 1,
554
- :tRCURLY, "}", :expr_endarg, 0, 0,
555
- :tSEMI, ";", :expr_beg, 0, 0,
556
-
557
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
558
- :tLCURLY, "{", :expr_beg, 0, 1,
559
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 1, # different
560
- :tSYMBOL, "c", :expr_end, 0, 1,
561
- :tRCURLY, "}", :expr_endarg, 0, 0)
562
-
563
- assert_lex("X = a { }; b { X :c }",
564
- s(:block,
565
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
566
- s(:iter,
567
- s(:call, nil, :b),
568
- 0,
569
- s(:call, nil, :X, s(:lit, :c)))),
570
-
571
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
572
- :tEQL, "=", :expr_beg, 0, 0,
573
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
574
- :tLCURLY, "{", :expr_beg, 0, 1,
575
- :tRCURLY, "}", :expr_endarg, 0, 0,
576
- :tSEMI, ";", :expr_beg, 0, 0,
577
-
578
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
579
- :tLCURLY, "{", :expr_beg, 0, 1,
580
- :tCONSTANT, "X", :expr_cmdarg, 0, 1, # same
581
- :tSYMBOL, "c", :expr_end, 0, 1,
582
- :tRCURLY, "}", :expr_endarg, 0, 0)
583
- end
584
-
585
- def test_yylex_lasgn_call_same_name
586
- assert_lex("a = b.c :d => 1",
587
- s(:lasgn, :a,
588
- s(:call, s(:call, nil, :b), :c,
589
- s(:hash, s(:lit, :d), s(:lit, 1)))),
590
-
591
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
592
- :tEQL, "=", :expr_beg, 0, 0,
593
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
594
- :tDOT, ".", :expr_dot, 0, 0,
595
- :tIDENTIFIER, "c", :expr_arg, 0, 0, # different
596
- :tSYMBOL, "d", :expr_end, 0, 0,
597
- :tASSOC, "=>", :expr_beg, 0, 0,
598
- :tINTEGER, 1, :expr_end, 0, 0)
599
-
600
- assert_lex("a = b.a :d => 1",
601
- s(:lasgn, :a,
602
- s(:call, s(:call, nil, :b), :a,
603
- s(:hash, s(:lit, :d), s(:lit, 1)))),
604
-
605
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
606
- :tEQL, "=", :expr_beg, 0, 0,
607
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
608
- :tDOT, ".", :expr_dot, 0, 0,
609
- :tIDENTIFIER, "a", :expr_arg, 0, 0, # same as lvar
610
- :tSYMBOL, "d", :expr_end, 0, 0,
611
- :tASSOC, "=>", :expr_beg, 0, 0,
612
- :tINTEGER, 1, :expr_end, 0, 0)
403
+ assert_lex3 "=>", nil, :tASSOC, "=>", EXPR_BEG
613
404
  end
614
405
 
615
406
  def test_yylex_back_ref
616
407
  assert_lex3("[$&, $`, $', $+]",
617
408
  nil,
618
- :tLBRACK, "[", :expr_beg,
619
- :tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
620
- :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
621
- :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
622
- :tBACK_REF, :+, :expr_end,
623
- :tRBRACK, "]", :expr_endarg)
409
+ :tLBRACK, "[", EXPR_PAR,
410
+ :tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
411
+ :tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
412
+ :tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
413
+ :tBACK_REF, :+, EXPR_END,
414
+ :tRBRACK, "]", EXPR_END)
624
415
  end
625
416
 
626
417
  def test_yylex_backslash
627
418
  assert_lex3("1 \\\n+ 2",
628
419
  nil,
629
- :tINTEGER, 1, :expr_end,
630
- :tPLUS, "+", :expr_beg,
631
- :tINTEGER, 2, :expr_end)
420
+ :tINTEGER, 1, EXPR_NUM,
421
+ :tPLUS, "+", EXPR_BEG,
422
+ :tINTEGER, 2, EXPR_NUM)
632
423
  end
633
424
 
634
425
  def test_yylex_backslash_bad
@@ -638,37 +429,37 @@ class TestRubyLexer < Minitest::Test
638
429
  def test_yylex_backtick
639
430
  assert_lex3("`ls`",
640
431
  nil,
641
- :tXSTRING_BEG, "`", :expr_beg,
642
- :tSTRING_CONTENT, "ls", :expr_beg,
643
- :tSTRING_END, "`", :expr_end)
432
+ :tXSTRING_BEG, "`", EXPR_BEG,
433
+ :tSTRING_CONTENT, "ls", EXPR_BEG,
434
+ :tSTRING_END, "`", EXPR_LIT)
644
435
  end
645
436
 
646
437
  def test_yylex_backtick_cmdarg
647
- self.lex_state = :expr_dot
438
+ self.lex_state = EXPR_DOT
648
439
 
649
440
  # \n ensures expr_cmd (TODO: why?)
650
- assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
441
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
651
442
  end
652
443
 
653
444
  def test_yylex_backtick_dot
654
- self.lex_state = :expr_dot
445
+ self.lex_state = EXPR_DOT
655
446
 
656
447
  assert_lex3("a.`(3)",
657
448
  nil,
658
- :tIDENTIFIER, "a", :expr_cmdarg,
659
- :tDOT, ".", :expr_dot,
660
- :tBACK_REF2, "`", :expr_arg,
661
- :tLPAREN2, "(", :expr_beg,
662
- :tINTEGER, 3, :expr_end,
663
- :tRPAREN, ")", :expr_endfn)
449
+ :tIDENTIFIER, "a", EXPR_CMDARG,
450
+ :tDOT, ".", EXPR_DOT,
451
+ :tBACK_REF2, "`", EXPR_ARG,
452
+ :tLPAREN2, "(", EXPR_PAR,
453
+ :tINTEGER, 3, EXPR_NUM,
454
+ :tRPAREN, ")", EXPR_ENDFN)
664
455
  end
665
456
 
666
457
  def test_yylex_backtick_method
667
- self.lex_state = :expr_fname
458
+ self.lex_state = EXPR_FNAME
668
459
 
669
460
  assert_lex3("`",
670
461
  nil,
671
- :tBACK_REF2, "`", :expr_end)
462
+ :tBACK_REF2, "`", EXPR_END)
672
463
  end
673
464
 
674
465
  def test_yylex_bad_char
@@ -676,24 +467,24 @@ class TestRubyLexer < Minitest::Test
676
467
  end
677
468
 
678
469
  def test_yylex_bang
679
- assert_lex3("!", nil, :tBANG, "!", :expr_beg)
470
+ assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
680
471
  end
681
472
 
682
473
  def test_yylex_bang_equals
683
- assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
474
+ assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
684
475
  end
685
476
 
686
477
  def test_yylex_bang_tilde
687
- assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
478
+ assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
688
479
  end
689
480
 
690
481
  def test_yylex_block_bug_1
691
482
  assert_lex3("a do end",
692
483
  s(:iter, s(:call, nil, :a), 0),
693
484
 
694
- :tIDENTIFIER, "a", :expr_cmdarg,
695
- :kDO, "do", :expr_beg,
696
- :kEND, "end", :expr_end)
485
+ :tIDENTIFIER, "a", EXPR_CMDARG,
486
+ :kDO, "do", EXPR_BEG,
487
+ :kEND, "end", EXPR_END)
697
488
  end
698
489
 
699
490
  def test_yylex_block_bug_2
@@ -702,70 +493,70 @@ class TestRubyLexer < Minitest::Test
702
493
  s(:lasgn, :a, s(:lit, 1)),
703
494
  s(:iter, s(:call, nil, :a), 0)),
704
495
 
705
- :tIDENTIFIER, "a", :expr_cmdarg,
706
- :tEQL, "=", :expr_beg,
707
- :tINTEGER, 1, :expr_end,
708
- :tNL, nil, :expr_beg,
709
- :tIDENTIFIER, "a", :expr_cmdarg,
710
- :kDO, "do", :expr_beg,
711
- :kEND, "end", :expr_end)
496
+ :tIDENTIFIER, "a", EXPR_CMDARG,
497
+ :tEQL, "=", EXPR_BEG,
498
+ :tINTEGER, 1, EXPR_NUM,
499
+ :tNL, nil, EXPR_BEG,
500
+ :tIDENTIFIER, "a", EXPR_CMDARG,
501
+ :kDO, "do", EXPR_BEG,
502
+ :kEND, "end", EXPR_END)
712
503
  end
713
504
 
714
505
  def test_yylex_block_bug_3
715
506
  assert_lex3("a { }",
716
507
  s(:iter, s(:call, nil, :a), 0),
717
508
 
718
- :tIDENTIFIER, "a", :expr_cmdarg, # verified
719
- :tLCURLY, "{", :expr_beg, # TODO: expr_beg|expr_label
720
- :tRCURLY, "}", :expr_endarg)
509
+ :tIDENTIFIER, "a", EXPR_CMDARG, # verified
510
+ :tLCURLY, "{", EXPR_PAR,
511
+ :tRCURLY, "}", EXPR_END)
721
512
  end
722
513
 
723
514
  def test_yylex_carat
724
- assert_lex3("^", nil, :tCARET, "^", :expr_beg)
515
+ assert_lex3("^", nil, :tCARET, "^", EXPR_BEG)
725
516
  end
726
517
 
727
518
  def test_yylex_carat_equals
728
- assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
519
+ assert_lex3("^=", nil, :tOP_ASGN, "^", EXPR_BEG)
729
520
  end
730
521
 
731
522
  def test_yylex_colon2
732
523
  assert_lex3("A::B",
733
524
  nil,
734
- :tCONSTANT, "A", :expr_cmdarg,
735
- :tCOLON2, "::", :expr_dot,
736
- :tCONSTANT, "B", :expr_arg)
525
+ :tCONSTANT, "A", EXPR_CMDARG,
526
+ :tCOLON2, "::", EXPR_DOT,
527
+ :tCONSTANT, "B", EXPR_ARG)
737
528
  end
738
529
 
739
530
  def test_yylex_colon2_argh
740
531
  assert_lex3("module X::Y\n c\nend",
741
532
  nil,
742
- :kMODULE, "module", :expr_beg,
743
- :tCONSTANT, "X", :expr_cmdarg,
744
- :tCOLON2, "::", :expr_dot,
745
- :tCONSTANT, "Y", :expr_arg,
746
- :tNL, nil, :expr_beg,
747
- :tIDENTIFIER, "c", :expr_cmdarg,
748
- :tNL, nil, :expr_beg,
749
- :kEND, "end", :expr_end)
533
+ :kMODULE, "module", EXPR_BEG,
534
+ :tCONSTANT, "X", EXPR_CMDARG,
535
+ :tCOLON2, "::", EXPR_DOT,
536
+ :tCONSTANT, "Y", EXPR_ARG,
537
+ :tNL, nil, EXPR_BEG,
538
+ :tIDENTIFIER, "c", EXPR_CMDARG,
539
+ :tNL, nil, EXPR_BEG,
540
+ :kEND, "end", EXPR_END)
750
541
  end
751
542
 
752
543
  def test_yylex_colon3
753
544
  assert_lex3("::Array",
754
545
  nil,
755
- :tCOLON3, "::", :expr_beg,
756
- :tCONSTANT, "Array", :expr_arg)
546
+ :tCOLON3, "::", EXPR_BEG,
547
+ :tCONSTANT, "Array", EXPR_ARG)
757
548
  end
758
549
 
759
550
  def test_yylex_comma
760
- assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
551
+ assert_lex3(",", nil, :tCOMMA, ",", EXPR_PAR)
761
552
  end
762
553
 
763
554
  def test_yylex_comment
764
555
  assert_lex3("1 # one\n# two\n2",
765
556
  nil,
766
- :tINTEGER, 1, :expr_end,
767
- :tNL, nil, :expr_beg,
768
- :tINTEGER, 2, :expr_end)
557
+ :tINTEGER, 1, EXPR_NUM,
558
+ :tNL, nil, EXPR_BEG,
559
+ :tINTEGER, 2, EXPR_NUM)
769
560
 
770
561
  assert_equal "# one\n# two\n", @lex.comments
771
562
  end
@@ -773,7 +564,7 @@ class TestRubyLexer < Minitest::Test
773
564
  def test_yylex_comment_begin
774
565
  assert_lex3("=begin\nblah\nblah\n=end\n42",
775
566
  nil,
776
- :tINTEGER, 42, :expr_end)
567
+ :tINTEGER, 42, EXPR_NUM)
777
568
 
778
569
  assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
779
570
  end
@@ -787,14 +578,14 @@ class TestRubyLexer < Minitest::Test
787
578
  def test_yylex_comment_begin_not_comment
788
579
  assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
789
580
  nil,
790
- :tIDENTIFIER, "beginfoo", :expr_cmdarg,
791
- :tEQL, "=", :expr_beg,
792
- :tINTEGER, 5, :expr_end,
793
- :tNL, nil, :expr_beg,
794
- :tIDENTIFIER, "p", :expr_cmdarg,
795
- :tIDENTIFIER, "x", :expr_arg,
796
- :tEQL, "=", :expr_beg,
797
- :tIDENTIFIER, "beginfoo", :expr_arg)
581
+ :tIDENTIFIER, "beginfoo", EXPR_CMDARG,
582
+ :tEQL, "=", EXPR_BEG,
583
+ :tINTEGER, 5, EXPR_NUM,
584
+ :tNL, nil, EXPR_BEG,
585
+ :tIDENTIFIER, "p", EXPR_CMDARG,
586
+ :tIDENTIFIER, "x", EXPR_ARG,
587
+ :tEQL, "=", EXPR_BEG,
588
+ :tIDENTIFIER, "beginfoo", EXPR_ARG)
798
589
  end
799
590
 
800
591
  def test_yylex_comment_begin_space
@@ -813,19 +604,63 @@ class TestRubyLexer < Minitest::Test
813
604
  assert_lex3("# comment", nil)
814
605
  end
815
606
 
816
- def test_yylex_constant
817
- assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
607
+ def test_yylex_const_call_same_name
608
+ assert_lex("X = a { }; b { f :c }",
609
+ s(:block,
610
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
611
+ s(:iter,
612
+ s(:call, nil, :b),
613
+ 0,
614
+ s(:call, nil, :f, s(:lit, :c)))),
615
+
616
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
617
+ :tEQL, "=", EXPR_BEG, 0, 0,
618
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
619
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
620
+ :tRCURLY, "}", EXPR_END, 0, 0,
621
+ :tSEMI, ";", EXPR_BEG, 0, 0,
622
+
623
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
624
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
625
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
626
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
627
+ :tRCURLY, "}", EXPR_END, 0, 0)
628
+
629
+ assert_lex("X = a { }; b { X :c }",
630
+ s(:block,
631
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
632
+ s(:iter,
633
+ s(:call, nil, :b),
634
+ 0,
635
+ s(:call, nil, :X, s(:lit, :c)))),
636
+
637
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
638
+ :tEQL, "=", EXPR_BEG, 0, 0,
639
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
640
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
641
+ :tRCURLY, "}", EXPR_END, 0, 0,
642
+ :tSEMI, ";", EXPR_BEG, 0, 0,
643
+
644
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
645
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
646
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
647
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
648
+ :tRCURLY, "}", EXPR_END, 0, 0)
649
+ end
650
+
651
+ def test_yylex_constant
652
+ assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", EXPR_CMDARG)
818
653
  end
819
654
 
820
655
  def test_yylex_constant_semi
821
656
  assert_lex3("ArgumentError;",
822
657
  nil,
823
- :tCONSTANT, "ArgumentError", :expr_cmdarg,
824
- :tSEMI, ";", :expr_beg)
658
+ :tCONSTANT, "ArgumentError", EXPR_CMDARG,
659
+ :tSEMI, ";", EXPR_BEG)
825
660
  end
826
661
 
827
662
  def test_yylex_cvar
828
- assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
663
+ assert_lex3("@@blah", nil, :tCVAR, "@@blah", EXPR_END)
829
664
  end
830
665
 
831
666
  def test_yylex_cvar_bad
@@ -835,75 +670,67 @@ class TestRubyLexer < Minitest::Test
835
670
  end
836
671
 
837
672
  def test_yylex_def_bad_name
838
- self.lex_state = :expr_fname
673
+ self.lex_state = EXPR_FNAME
839
674
  refute_lex("def [ ", :kDEF, "def")
840
675
  end
841
676
 
842
677
  def test_yylex_div
843
678
  assert_lex3("a / 2",
844
679
  nil,
845
- :tIDENTIFIER, "a", :expr_cmdarg,
846
- :tDIVIDE, "/", :expr_beg,
847
- :tINTEGER, 2, :expr_end)
680
+ :tIDENTIFIER, "a", EXPR_CMDARG,
681
+ :tDIVIDE, "/", EXPR_BEG,
682
+ :tINTEGER, 2, EXPR_NUM)
848
683
  end
849
684
 
850
685
  def test_yylex_div_equals
851
686
  assert_lex3("a /= 2",
852
687
  nil,
853
- :tIDENTIFIER, "a", :expr_cmdarg,
854
- :tOP_ASGN, "/", :expr_beg,
855
- :tINTEGER, 2, :expr_end)
688
+ :tIDENTIFIER, "a", EXPR_CMDARG,
689
+ :tOP_ASGN, "/", EXPR_BEG,
690
+ :tINTEGER, 2, EXPR_NUM)
856
691
  end
857
692
 
858
693
  def test_yylex_do
859
694
  assert_lex3("x do 42 end",
860
695
  nil,
861
- :tIDENTIFIER, "x", :expr_cmdarg,
862
- :kDO, "do", :expr_beg,
863
- :tINTEGER, 42, :expr_end,
864
- :kEND, "end", :expr_end)
696
+ :tIDENTIFIER, "x", EXPR_CMDARG,
697
+ :kDO, "do", EXPR_BEG,
698
+ :tINTEGER, 42, EXPR_NUM,
699
+ :kEND, "end", EXPR_END)
865
700
  end
866
701
 
867
702
  def test_yylex_do_block
868
- self.lex_state = :expr_endarg
703
+ self.lex_state = EXPR_ENDARG
869
704
 
870
705
  assert_lex3("x.y do 42 end",
871
706
  nil,
872
- :tIDENTIFIER, "x", :expr_end,
873
- :tDOT, ".", :expr_dot,
874
- :tIDENTIFIER, "y", :expr_arg,
875
- :kDO_BLOCK, "do", :expr_beg,
876
- :tINTEGER, 42, :expr_end,
877
- :kEND, "end", :expr_end) do
707
+ :tIDENTIFIER, "x", EXPR_END,
708
+ :tDOT, ".", EXPR_DOT,
709
+ :tIDENTIFIER, "y", EXPR_ARG,
710
+ :kDO_BLOCK, "do", EXPR_BEG,
711
+ :tINTEGER, 42, EXPR_NUM,
712
+ :kEND, "end", EXPR_END) do
878
713
  @lex.cmdarg.push true
879
714
  end
880
715
  end
881
716
 
882
717
  def test_yylex_do_block2
883
- self.lex_state = :expr_endarg
718
+ self.lex_state = EXPR_ENDARG
884
719
 
885
720
  assert_lex3("do 42 end",
886
721
  nil,
887
- :kDO_BLOCK, "do", :expr_beg,
888
- :tINTEGER, 42, :expr_end,
889
- :kEND, "end", :expr_end)
890
- end
891
-
892
- def test_yylex_is_your_spacebar_broken?
893
- assert_lex3(":a!=:b",
894
- nil,
895
- :tSYMBOL, "a", :expr_end,
896
- :tNEQ, "!=", :expr_beg,
897
- :tSYMBOL, "b", :expr_end)
722
+ :kDO, "do", EXPR_BEG,
723
+ :tINTEGER, 42, EXPR_NUM,
724
+ :kEND, "end", EXPR_END)
898
725
  end
899
726
 
900
727
  def test_yylex_do_cond
901
728
  assert_lex3("x do 42 end",
902
729
  nil,
903
- :tIDENTIFIER, "x", :expr_cmdarg,
904
- :kDO_COND, "do", :expr_beg,
905
- :tINTEGER, 42, :expr_end,
906
- :kEND, "end", :expr_end) do
730
+ :tIDENTIFIER, "x", EXPR_CMDARG,
731
+ :kDO_COND, "do", EXPR_BEG,
732
+ :tINTEGER, 42, EXPR_NUM,
733
+ :kEND, "end", EXPR_END) do
907
734
  @lex.cond.push true
908
735
  end
909
736
  end
@@ -914,40 +741,40 @@ class TestRubyLexer < Minitest::Test
914
741
  end
915
742
 
916
743
  def test_yylex_dollar_eos
917
- assert_lex3("$", nil, "$", "$", :expr_end) # FIX: wtf is this?!?
744
+ assert_lex3("$", nil, "$", "$", EXPR_END) # FIX: wtf is this?!?
918
745
  end
919
746
 
920
747
  def test_yylex_dot # HINT message sends
921
- assert_lex3(".", nil, :tDOT, ".", :expr_dot)
748
+ assert_lex3(".", nil, :tDOT, ".", EXPR_DOT)
922
749
  end
923
750
 
924
751
  def test_yylex_dot2
925
- assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
752
+ assert_lex3("..", nil, :tDOT2, "..", EXPR_BEG)
926
753
  end
927
754
 
928
755
  def test_yylex_dot3
929
- assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
756
+ assert_lex3("...", nil, :tDOT3, "...", EXPR_BEG)
930
757
  end
931
758
 
932
759
  def test_yylex_equals
933
760
  # FIX: this sucks
934
- assert_lex3("=", nil, :tEQL, "=", :expr_beg)
761
+ assert_lex3("=", nil, :tEQL, "=", EXPR_BEG)
935
762
  end
936
763
 
937
764
  def test_yylex_equals2
938
- assert_lex3("==", nil, :tEQ, "==", :expr_beg)
765
+ assert_lex3("==", nil, :tEQ, "==", EXPR_BEG)
939
766
  end
940
767
 
941
768
  def test_yylex_equals3
942
- assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
769
+ assert_lex3("===", nil, :tEQQ, "===", EXPR_BEG)
943
770
  end
944
771
 
945
772
  def test_yylex_equals_tilde
946
- assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
773
+ assert_lex3("=~", nil, :tMATCH, "=~", EXPR_BEG)
947
774
  end
948
775
 
949
776
  def test_yylex_float
950
- assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
777
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
951
778
  end
952
779
 
953
780
  def test_yylex_float_bad_no_underscores
@@ -965,55 +792,48 @@ class TestRubyLexer < Minitest::Test
965
792
  def test_yylex_float_call
966
793
  assert_lex3("1.0.to_s",
967
794
  nil,
968
- :tFLOAT, 1.0, :expr_end,
969
- :tDOT, ".", :expr_dot,
970
- :tIDENTIFIER, "to_s", :expr_arg)
795
+ :tFLOAT, 1.0, EXPR_NUM,
796
+ :tDOT, ".", EXPR_DOT,
797
+ :tIDENTIFIER, "to_s", EXPR_ARG)
971
798
  end
972
799
 
973
800
  def test_yylex_float_dot_E
974
801
  assert_lex3("1.0E10",
975
802
  nil,
976
- :tFLOAT, 10000000000.0, :expr_end)
803
+ :tFLOAT, 10000000000.0, EXPR_NUM)
977
804
  end
978
805
 
979
806
  def test_yylex_float_dot_E_neg
980
807
  assert_lex3("-1.0E10",
981
808
  nil,
982
- :tUMINUS_NUM, "-", :expr_beg,
983
- :tFLOAT, 10000000000.0, :expr_end)
809
+ :tUMINUS_NUM, "-", EXPR_BEG,
810
+ :tFLOAT, 10000000000.0, EXPR_NUM)
984
811
  end
985
812
 
986
813
  def test_yylex_float_dot_e
987
814
  assert_lex3("1.0e10",
988
815
  nil,
989
- :tFLOAT, 10000000000.0, :expr_end)
816
+ :tFLOAT, 10000000000.0, EXPR_NUM)
990
817
  end
991
818
 
992
819
  def test_yylex_float_dot_e_neg
993
820
  assert_lex3("-1.0e10",
994
821
  nil,
995
- :tUMINUS_NUM, "-", :expr_beg,
996
- :tFLOAT, 10000000000.0, :expr_end)
822
+ :tUMINUS_NUM, "-", EXPR_BEG,
823
+ :tFLOAT, 10000000000.0, EXPR_NUM)
997
824
  end
998
825
 
999
826
  def test_yylex_float_e
1000
827
  assert_lex3("1e10",
1001
828
  nil,
1002
- :tFLOAT, 10000000000.0, :expr_end)
829
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1003
830
  end
1004
831
 
1005
832
  def test_yylex_float_e_bad_double_e
1006
833
  assert_lex3("1e2e3",
1007
834
  nil,
1008
- :tFLOAT, 100, :expr_end,
1009
- :tIDENTIFIER, "e3", :expr_end)
1010
- end
1011
-
1012
- def test_yylex_float_if_modifier
1013
- assert_lex3("1e2if",
1014
- nil,
1015
- :tFLOAT, 100, :expr_end,
1016
- :kIF_MOD, "if", :expr_beg)
835
+ :tFLOAT, 100, EXPR_NUM,
836
+ :tIDENTIFIER, "e3", EXPR_END)
1017
837
  end
1018
838
 
1019
839
  def test_yylex_float_e_bad_trailing_underscore
@@ -1021,198 +841,258 @@ class TestRubyLexer < Minitest::Test
1021
841
  end
1022
842
 
1023
843
  def test_yylex_float_e_minus
1024
- assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
844
+ assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, EXPR_NUM)
1025
845
  end
1026
846
 
1027
847
  def test_yylex_float_e_neg
1028
848
  assert_lex3("-1e10",
1029
849
  nil,
1030
- :tUMINUS_NUM, "-", :expr_beg,
1031
- :tFLOAT, 10000000000.0, :expr_end)
850
+ :tUMINUS_NUM, "-", EXPR_BEG,
851
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1032
852
  end
1033
853
 
1034
854
  def test_yylex_float_e_neg_minus
1035
855
  assert_lex3("-1e-10",
1036
856
  nil,
1037
- :tUMINUS_NUM, "-", :expr_beg,
1038
- :tFLOAT, 1.0e-10, :expr_end)
857
+ :tUMINUS_NUM, "-", EXPR_BEG,
858
+ :tFLOAT, 1.0e-10, EXPR_NUM)
1039
859
  end
1040
860
 
1041
861
  def test_yylex_float_e_neg_plus
1042
862
  assert_lex3("-1e+10",
1043
863
  nil,
1044
- :tUMINUS_NUM, "-", :expr_beg,
1045
- :tFLOAT, 10000000000.0, :expr_end)
864
+ :tUMINUS_NUM, "-", EXPR_BEG,
865
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1046
866
  end
1047
867
 
1048
868
  def test_yylex_float_e_plus
1049
- assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
869
+ assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, EXPR_NUM)
1050
870
  end
1051
871
 
1052
872
  def test_yylex_float_e_zero
1053
- assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
873
+ assert_lex3("0e0", nil, :tFLOAT, 0.0, EXPR_NUM)
874
+ end
875
+
876
+ def test_yylex_float_if_modifier
877
+ assert_lex3("1e2if",
878
+ nil,
879
+ :tFLOAT, 100, EXPR_NUM,
880
+ :kIF_MOD, "if", EXPR_PAR)
1054
881
  end
1055
882
 
1056
883
  def test_yylex_float_neg
1057
884
  assert_lex3("-1.0",
1058
885
  nil,
1059
- :tUMINUS_NUM, "-", :expr_beg,
1060
- :tFLOAT, 1.0, :expr_end)
886
+ :tUMINUS_NUM, "-", EXPR_BEG,
887
+ :tFLOAT, 1.0, EXPR_NUM)
1061
888
  end
1062
889
 
1063
890
  def test_yylex_ge
1064
891
  assert_lex3("a >= 2",
1065
892
  nil,
1066
- :tIDENTIFIER, "a", :expr_cmdarg,
1067
- :tGEQ, ">=", :expr_beg,
1068
- :tINTEGER, 2, :expr_end)
893
+ :tIDENTIFIER, "a", EXPR_CMDARG,
894
+ :tGEQ, ">=", EXPR_BEG,
895
+ :tINTEGER, 2, EXPR_NUM)
1069
896
  end
1070
897
 
1071
898
  def test_yylex_global
1072
- assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
899
+ assert_lex3("$blah", nil, :tGVAR, "$blah", EXPR_END)
1073
900
  end
1074
901
 
1075
902
  def test_yylex_global_backref
1076
- self.lex_state = :expr_fname
903
+ self.lex_state = EXPR_FNAME
1077
904
 
1078
- assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
905
+ assert_lex3("$`", nil, :tGVAR, "$`", EXPR_END)
1079
906
  end
1080
907
 
1081
908
  def test_yylex_global_dash_nothing
1082
- assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
909
+ assert_lex3("$- ", nil, :tGVAR, "$-", EXPR_END)
1083
910
  end
1084
911
 
1085
912
  def test_yylex_global_dash_something
1086
- assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
913
+ assert_lex3("$-x", nil, :tGVAR, "$-x", EXPR_END)
1087
914
  end
1088
915
 
1089
916
  def test_yylex_global_number
1090
- self.lex_state = :expr_fname
917
+ self.lex_state = EXPR_FNAME
1091
918
 
1092
- assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
919
+ assert_lex3("$1", nil, :tGVAR, "$1", EXPR_END)
1093
920
  end
1094
921
 
1095
922
  def test_yylex_global_number_big
1096
- self.lex_state = :expr_fname
923
+ self.lex_state = EXPR_FNAME
924
+
925
+ assert_lex3("$1234", nil, :tGVAR, "$1234", EXPR_END)
926
+ end
1097
927
 
1098
- assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
928
+ def test_yylex_global_I_have_no_words
929
+ assert_lex3("$x\xE2\x80\x8B = 42", # zero width space?!?!?
930
+ nil,
931
+ :tGVAR, "$x\xE2\x80\x8B", EXPR_END,
932
+ :tEQL, "=", EXPR_BEG,
933
+ :tINTEGER, 42, EXPR_NUM)
1099
934
  end
1100
935
 
1101
936
  def test_yylex_global_other
1102
937
  assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
1103
938
  nil,
1104
- :tLBRACK, "[", :expr_beg,
1105
- :tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
1106
- :tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
1107
- :tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
1108
- :tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
1109
- :tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
1110
- :tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
1111
- :tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
1112
- :tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
1113
- :tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
1114
- :tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
1115
- :tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
1116
- :tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
1117
- :tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
1118
- :tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
1119
- :tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
1120
- :tGVAR, "$\"", :expr_end,
1121
- :tRBRACK, "]", :expr_endarg)
939
+ :tLBRACK, "[", EXPR_PAR,
940
+ :tGVAR, "$~", EXPR_END, :tCOMMA, ",", EXPR_PAR,
941
+ :tGVAR, "$*", EXPR_END, :tCOMMA, ",", EXPR_PAR,
942
+ :tGVAR, "$$", EXPR_END, :tCOMMA, ",", EXPR_PAR,
943
+ :tGVAR, "$?", EXPR_END, :tCOMMA, ",", EXPR_PAR,
944
+ :tGVAR, "$!", EXPR_END, :tCOMMA, ",", EXPR_PAR,
945
+ :tGVAR, "$@", EXPR_END, :tCOMMA, ",", EXPR_PAR,
946
+ :tGVAR, "$/", EXPR_END, :tCOMMA, ",", EXPR_PAR,
947
+ :tGVAR, "$\\", EXPR_END, :tCOMMA, ",", EXPR_PAR,
948
+ :tGVAR, "$;", EXPR_END, :tCOMMA, ",", EXPR_PAR,
949
+ :tGVAR, "$,", EXPR_END, :tCOMMA, ",", EXPR_PAR,
950
+ :tGVAR, "$.", EXPR_END, :tCOMMA, ",", EXPR_PAR,
951
+ :tGVAR, "$=", EXPR_END, :tCOMMA, ",", EXPR_PAR,
952
+ :tGVAR, "$:", EXPR_END, :tCOMMA, ",", EXPR_PAR,
953
+ :tGVAR, "$<", EXPR_END, :tCOMMA, ",", EXPR_PAR,
954
+ :tGVAR, "$>", EXPR_END, :tCOMMA, ",", EXPR_PAR,
955
+ :tGVAR, "$\"", EXPR_END,
956
+ :tRBRACK, "]", EXPR_END)
1122
957
  end
1123
958
 
1124
959
  def test_yylex_global_underscore
1125
- assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
960
+ assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
1126
961
  end
1127
962
 
1128
963
  def test_yylex_global_wierd
1129
- assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
964
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
1130
965
  end
1131
966
 
1132
967
  def test_yylex_global_zero
1133
- assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
968
+ assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
1134
969
  end
1135
970
 
1136
971
  def test_yylex_gt
1137
972
  assert_lex3("a > 2",
1138
973
  nil,
1139
- :tIDENTIFIER, "a", :expr_cmdarg,
1140
- :tGT, ">", :expr_beg,
1141
- :tINTEGER, 2, :expr_end)
974
+ :tIDENTIFIER, "a", EXPR_CMDARG,
975
+ :tGT, ">", EXPR_BEG,
976
+ :tINTEGER, 2, EXPR_NUM)
1142
977
  end
1143
978
 
1144
- def test_yylex_heredoc_backtick
1145
- assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1146
- nil,
1147
- :tIDENTIFIER, "a", :expr_cmdarg,
1148
- :tEQL, "=", :expr_beg,
1149
- :tXSTRING_BEG, "`", :expr_beg,
1150
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1151
- :tSTRING_END, "EOF", :expr_end,
1152
- :tNL, nil, :expr_beg)
979
+ def test_yylex_hash_colon
980
+ assert_lex("{a:1}",
981
+ s(:hash, s(:lit, :a), s(:lit, 1)),
982
+
983
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
984
+ :tLABEL, "a", EXPR_LAB, 0, 1,
985
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
986
+ :tRCURLY, "}", EXPR_END, 0, 0)
1153
987
  end
1154
988
 
1155
- def test_yylex_heredoc_double
1156
- assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1157
- nil,
1158
- :tIDENTIFIER, "a", :expr_cmdarg,
1159
- :tEQL, "=", :expr_beg,
1160
- :tSTRING_BEG, "\"", :expr_beg,
1161
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1162
- :tSTRING_END, "EOF", :expr_end,
1163
- :tNL, nil, :expr_beg)
989
+ def test_yylex_hash_colon_double_quoted_symbol
990
+ assert_lex('{"abc": :b}',
991
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
992
+
993
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
994
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
995
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
996
+ :tRCURLY, "}", EXPR_END, 0, 0)
1164
997
  end
1165
998
 
1166
- def test_yylex_heredoc_double_dash
1167
- assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1168
- nil,
1169
- :tIDENTIFIER, "a", :expr_cmdarg,
1170
- :tEQL, "=", :expr_beg,
1171
- :tSTRING_BEG, "\"", :expr_beg,
1172
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1173
- :tSTRING_END, "EOF", :expr_end,
1174
- :tNL, nil, :expr_beg)
999
+ def test_yylex_hash_colon_double_quoted_symbol_22
1000
+ setup_lexer_class RubyParser::V22
1001
+
1002
+ assert_lex('{"abc": :b}',
1003
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1004
+
1005
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1006
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1007
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1008
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1175
1009
  end
1176
1010
 
1177
- def test_yylex_heredoc_double_squiggly
1178
- setup_lexer_class Ruby23Parser
1011
+ def test_yylex_hash_colon_double_quoted_with_escapes
1012
+ assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
1013
+ nil,
1179
1014
 
1180
- assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1181
- nil,
1182
- :tIDENTIFIER, "a", :expr_cmdarg,
1183
- :tEQL, "=", :expr_beg,
1184
- :tSTRING_BEG, "\"", :expr_beg,
1185
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1186
- :tSTRING_END, "EOF", :expr_end,
1187
- :tNL, nil, :expr_beg)
1015
+ :tLBRACE, "{", EXPR_PAR,
1016
+ :tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
1017
+ :tINTEGER, 1, EXPR_NUM,
1018
+ :tRCURLY, "}", EXPR_END)
1188
1019
  end
1189
1020
 
1190
- # mri handles tabs in a pretty specific way:
1191
- # https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
1192
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
1193
- setup_lexer_class Ruby23Parser
1021
+ def test_yylex_hash_colon_quoted_22
1022
+ setup_lexer_class RubyParser::V22
1194
1023
 
1195
- assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
1024
+ assert_lex("{'a':1}",
1025
+ s(:hash, s(:lit, :a), s(:lit, 1)),
1026
+
1027
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1028
+ :tLABEL, "a", EXPR_LAB, 0, 1,
1029
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
1030
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1031
+ end
1032
+
1033
+ def test_yylex_hash_colon_quoted_symbol
1034
+ assert_lex("{'abc': :b}",
1035
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1036
+
1037
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1038
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1039
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1040
+ :tRCURLY, "}", EXPR_END, 0, 0)
1041
+ end
1042
+
1043
+ def test_yylex_hash_colon_quoted_symbol_22
1044
+ setup_lexer_class RubyParser::V22
1045
+
1046
+ assert_lex("{'abc': :b}",
1047
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1048
+
1049
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1050
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1051
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1052
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1053
+ end
1054
+
1055
+ def test_yylex_hash_colon_quoted_with_escapes
1056
+ assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
1057
+ nil,
1058
+
1059
+ :tLBRACE, "{", EXPR_PAR,
1060
+ :tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
1061
+ :tINTEGER, 1, EXPR_NUM,
1062
+ :tRCURLY, "}", EXPR_END)
1063
+ end
1064
+
1065
+ def test_yylex_heredoc_backtick
1066
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1196
1067
  nil,
1197
- :tIDENTIFIER, "a", :expr_cmdarg,
1198
- :tEQL, "=", :expr_beg,
1199
- :tSTRING_BEG, "\"", :expr_beg,
1200
- :tSTRING_CONTENT, "blah blah\n\tblah blah\n", :expr_beg,
1201
- :tSTRING_END, "EOF", :expr_end,
1202
- :tNL, nil, :expr_beg)
1068
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1069
+ :tEQL, "=", EXPR_BEG,
1070
+ :tXSTRING_BEG, "`", EXPR_BEG,
1071
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1072
+ :tSTRING_END, "EOF", EXPR_LIT,
1073
+ :tNL, nil, EXPR_BEG)
1203
1074
  end
1204
1075
 
1205
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
1206
- setup_lexer_class Ruby23Parser
1076
+ def test_yylex_heredoc_double
1077
+ assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1078
+ nil,
1079
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1080
+ :tEQL, "=", EXPR_BEG,
1081
+ :tSTRING_BEG, "\"", EXPR_BEG,
1082
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1083
+ :tSTRING_END, "EOF", EXPR_LIT,
1084
+ :tNL, nil, EXPR_BEG)
1085
+ end
1207
1086
 
1208
- assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
1087
+ def test_yylex_heredoc_double_dash
1088
+ assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1209
1089
  nil,
1210
- :tIDENTIFIER, "a", :expr_cmdarg,
1211
- :tEQL, "=", :expr_beg,
1212
- :tSTRING_BEG, "\"", :expr_beg,
1213
- :tSTRING_CONTENT, "blah blah\n blah blah\n", :expr_beg,
1214
- :tSTRING_END, "EOF", :expr_end,
1215
- :tNL, nil, :expr_beg)
1090
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1091
+ :tEQL, "=", EXPR_BEG,
1092
+ :tSTRING_BEG, "\"", EXPR_BEG,
1093
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1094
+ :tSTRING_END, "EOF", EXPR_LIT,
1095
+ :tNL, nil, EXPR_BEG)
1216
1096
  end
1217
1097
 
1218
1098
  def test_yylex_heredoc_double_eos
@@ -1230,41 +1110,56 @@ class TestRubyLexer < Minitest::Test
1230
1110
  end
1231
1111
 
1232
1112
  def test_yylex_heredoc_double_interp
1233
- assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
1234
- nil,
1235
- :tIDENTIFIER, "a", :expr_cmdarg,
1236
- :tEQL, "=", :expr_beg,
1237
- :tSTRING_BEG, "\"", :expr_beg,
1238
- :tSTRING_CONTENT, "#x a ", :expr_beg,
1239
- :tSTRING_DVAR, "\#@", :expr_beg,
1240
- :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
1241
- :tSTRING_DVAR, "\#$", :expr_beg,
1242
- :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
1243
- :tSTRING_DBEG, "\#{", :expr_beg,
1244
- :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
1245
- :tSTRING_END, "EOF", :expr_end,
1246
- :tNL, nil, :expr_beg)
1113
+ assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#@@d \#{3} \nEOF\n\n",
1114
+ nil,
1115
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1116
+ :tEQL, "=", EXPR_BEG,
1117
+ :tSTRING_BEG, "\"", EXPR_BEG,
1118
+ :tSTRING_CONTENT, "#x a ", EXPR_BEG,
1119
+ :tSTRING_DVAR, "#", EXPR_BEG,
1120
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG, # HUH?
1121
+ :tSTRING_DVAR, "#", EXPR_BEG,
1122
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG, # HUH?
1123
+ :tSTRING_DVAR, "#", EXPR_BEG,
1124
+ :tSTRING_CONTENT, "@@d ", EXPR_BEG, # HUH?
1125
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1126
+ :tSTRING_CONTENT, "3} \n", EXPR_BEG,
1127
+ :tSTRING_END, "EOF", EXPR_LIT,
1128
+ :tNL, nil, EXPR_BEG)
1129
+ end
1130
+
1131
+ def test_yylex_heredoc_double_squiggly
1132
+ setup_lexer_class Ruby23Parser
1133
+
1134
+ assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1135
+ nil,
1136
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1137
+ :tEQL, "=", EXPR_BEG,
1138
+ :tSTRING_BEG, "\"", EXPR_BEG,
1139
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1140
+ :tSTRING_END, "EOF", EXPR_LIT,
1141
+ :tNL, nil, EXPR_BEG)
1247
1142
  end
1248
1143
 
1249
1144
  def test_yylex_heredoc_empty
1250
1145
  assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1251
1146
  nil,
1252
- :tSTRING_BEG, "\"", :expr_beg,
1253
- :tSTRING_DBEG, "\#{", :expr_beg,
1254
- :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
1255
- :tSTRING_END, "", :expr_end,
1256
- :tNL, nil, :expr_beg)
1147
+ :tSTRING_BEG, "\"", EXPR_BEG,
1148
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1149
+ :tSTRING_CONTENT, "x}\nblah2\n", EXPR_BEG,
1150
+ :tSTRING_END, "", EXPR_LIT,
1151
+ :tNL, nil, EXPR_BEG)
1257
1152
  end
1258
1153
 
1259
1154
  def test_yylex_heredoc_none
1260
1155
  assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
1261
1156
  nil,
1262
- :tIDENTIFIER, "a", :expr_cmdarg,
1263
- :tEQL, "=", :expr_beg,
1264
- :tSTRING_BEG, "\"", :expr_beg,
1265
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1266
- :tSTRING_END, "EOF", :expr_end,
1267
- :tNL, nil, :expr_beg)
1157
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1158
+ :tEQL, "=", EXPR_BEG,
1159
+ :tSTRING_BEG, "\"", EXPR_BEG,
1160
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1161
+ :tSTRING_END, "EOF", EXPR_LIT,
1162
+ :tNL, nil, EXPR_BEG)
1268
1163
  end
1269
1164
 
1270
1165
  def test_yylex_heredoc_none_bad_eos
@@ -1277,12 +1172,12 @@ class TestRubyLexer < Minitest::Test
1277
1172
  def test_yylex_heredoc_none_dash
1278
1173
  assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
1279
1174
  nil,
1280
- :tIDENTIFIER, "a", :expr_cmdarg,
1281
- :tEQL, "=", :expr_beg,
1282
- :tSTRING_BEG, "\"", :expr_beg,
1283
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1284
- :tSTRING_END, "EOF", :expr_end,
1285
- :tNL, nil, :expr_beg)
1175
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1176
+ :tEQL, "=", EXPR_BEG,
1177
+ :tSTRING_BEG, "\"", EXPR_BEG,
1178
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1179
+ :tSTRING_END, "EOF", EXPR_LIT,
1180
+ :tNL, nil, EXPR_BEG)
1286
1181
  end
1287
1182
 
1288
1183
  def test_yylex_heredoc_none_squiggly
@@ -1290,23 +1185,23 @@ class TestRubyLexer < Minitest::Test
1290
1185
 
1291
1186
  assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
1292
1187
  nil,
1293
- :tIDENTIFIER, "a", :expr_cmdarg,
1294
- :tEQL, "=", :expr_beg,
1295
- :tSTRING_BEG, "\"", :expr_beg,
1296
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1297
- :tSTRING_END, "EOF", :expr_end,
1298
- :tNL, nil, :expr_beg)
1188
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1189
+ :tEQL, "=", EXPR_BEG,
1190
+ :tSTRING_BEG, "\"", EXPR_BEG,
1191
+ :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1192
+ :tSTRING_END, "EOF", EXPR_LIT,
1193
+ :tNL, nil, EXPR_BEG)
1299
1194
  end
1300
1195
 
1301
1196
  def test_yylex_heredoc_single
1302
1197
  assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1303
1198
  nil,
1304
- :tIDENTIFIER, "a", :expr_cmdarg,
1305
- :tEQL, "=", :expr_beg,
1306
- :tSTRING_BEG, "\"", :expr_beg,
1307
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1308
- :tSTRING_END, "EOF", :expr_end,
1309
- :tNL, nil, :expr_beg)
1199
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1200
+ :tEQL, "=", EXPR_BEG,
1201
+ :tSTRING_BEG, "\"", EXPR_BEG,
1202
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1203
+ :tSTRING_END, "EOF", EXPR_LIT,
1204
+ :tNL, nil, EXPR_BEG)
1310
1205
  end
1311
1206
 
1312
1207
  def test_yylex_heredoc_single_bad_eos_body
@@ -1340,12 +1235,12 @@ class TestRubyLexer < Minitest::Test
1340
1235
  def test_yylex_heredoc_single_dash
1341
1236
  assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
1342
1237
  nil,
1343
- :tIDENTIFIER, "a", :expr_cmdarg,
1344
- :tEQL, "=", :expr_beg,
1345
- :tSTRING_BEG, "\"", :expr_beg,
1346
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1347
- :tSTRING_END, "EOF", :expr_end,
1348
- :tNL, nil, :expr_beg)
1238
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1239
+ :tEQL, "=", EXPR_BEG,
1240
+ :tSTRING_BEG, "\"", EXPR_BEG,
1241
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1242
+ :tSTRING_END, "EOF", EXPR_LIT,
1243
+ :tNL, nil, EXPR_BEG)
1349
1244
  end
1350
1245
 
1351
1246
  def test_yylex_heredoc_single_squiggly
@@ -1353,24 +1248,24 @@ class TestRubyLexer < Minitest::Test
1353
1248
 
1354
1249
  assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
1355
1250
  nil,
1356
- :tIDENTIFIER, "a", :expr_cmdarg,
1357
- :tEQL, "=", :expr_beg,
1358
- :tSTRING_BEG, "\"", :expr_beg,
1359
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1360
- :tSTRING_END, "EOF", :expr_end,
1361
- :tNL, nil, :expr_beg)
1251
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1252
+ :tEQL, "=", EXPR_BEG,
1253
+ :tSTRING_BEG, "\"", EXPR_BEG,
1254
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1255
+ :tSTRING_END, "EOF", EXPR_LIT,
1256
+ :tNL, nil, EXPR_BEG)
1362
1257
  end
1363
1258
 
1364
1259
  def test_yylex_identifier
1365
1260
  assert_lex3("identifier",
1366
1261
  nil,
1367
- :tIDENTIFIER, "identifier", :expr_cmdarg)
1262
+ :tIDENTIFIER, "identifier", EXPR_CMDARG)
1368
1263
  end
1369
1264
 
1370
1265
  def test_yylex_identifier_bang
1371
1266
  assert_lex3("identifier!",
1372
1267
  nil,
1373
- :tFID, "identifier!", :expr_cmdarg)
1268
+ :tFID, "identifier!", EXPR_CMDARG)
1374
1269
  end
1375
1270
 
1376
1271
  def test_yylex_identifier_cmp
@@ -1380,57 +1275,57 @@ class TestRubyLexer < Minitest::Test
1380
1275
  def test_yylex_identifier_def__20
1381
1276
  setup_lexer_class RubyParser::V20
1382
1277
 
1383
- assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1278
+ assert_lex_fname "identifier", :tIDENTIFIER, EXPR_ENDFN
1384
1279
  end
1385
1280
 
1386
1281
  def test_yylex_identifier_eh
1387
- assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
1388
- end
1389
-
1390
- def test_yylex_identifier_equals_arrow
1391
- assert_lex3(":blah==>",
1392
- nil,
1393
- :tSYMBOL, "blah=", :expr_end,
1394
- :tASSOC, "=>", :expr_beg)
1282
+ assert_lex3("identifier?", nil, :tFID, "identifier?", EXPR_CMDARG)
1395
1283
  end
1396
1284
 
1397
1285
  def test_yylex_identifier_equals3
1398
1286
  assert_lex3(":a===b",
1399
1287
  nil,
1400
- :tSYMBOL, "a", :expr_end,
1401
- :tEQQ, "===", :expr_beg,
1402
- :tIDENTIFIER, "b", :expr_arg)
1288
+ :tSYMBOL, "a", EXPR_LIT,
1289
+ :tEQQ, "===", EXPR_BEG,
1290
+ :tIDENTIFIER, "b", EXPR_ARG)
1403
1291
  end
1404
1292
 
1405
- def test_yylex_identifier_equals_equals_arrow
1406
- assert_lex3(":a==>b",
1293
+ def test_yylex_identifier_equals_arrow
1294
+ assert_lex3(":blah==>",
1407
1295
  nil,
1408
- :tSYMBOL, "a=", :expr_end,
1409
- :tASSOC, "=>", :expr_beg,
1410
- :tIDENTIFIER, "b", :expr_arg)
1296
+ :tSYMBOL, "blah=", EXPR_LIT,
1297
+ :tASSOC, "=>", EXPR_BEG)
1411
1298
  end
1412
1299
 
1413
1300
  def test_yylex_identifier_equals_caret
1414
1301
  assert_lex_fname "^", :tCARET
1415
1302
  end
1416
1303
 
1304
+ def test_yylex_identifier_equals_def2
1305
+ assert_lex_fname "==", :tEQ
1306
+ end
1307
+
1417
1308
  def test_yylex_identifier_equals_def__20
1418
1309
  setup_lexer_class RubyParser::V20
1419
1310
 
1420
- assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1311
+ assert_lex_fname "identifier=", :tIDENTIFIER, EXPR_ENDFN
1421
1312
  end
1422
1313
 
1423
- def test_yylex_identifier_equals_def2
1424
- assert_lex_fname "==", :tEQ
1314
+ def test_yylex_identifier_equals_equals_arrow
1315
+ assert_lex3(":a==>b",
1316
+ nil,
1317
+ :tSYMBOL, "a=", EXPR_LIT,
1318
+ :tASSOC, "=>", EXPR_BEG,
1319
+ :tIDENTIFIER, "b", EXPR_ARG)
1425
1320
  end
1426
1321
 
1427
1322
  def test_yylex_identifier_equals_expr
1428
- self.lex_state = :expr_dot
1323
+ self.lex_state = EXPR_DOT
1429
1324
  assert_lex3("y = arg",
1430
1325
  nil,
1431
- :tIDENTIFIER, "y", :expr_cmdarg,
1432
- :tEQL, "=", :expr_beg,
1433
- :tIDENTIFIER, "arg", :expr_arg)
1326
+ :tIDENTIFIER, "y", EXPR_CMDARG,
1327
+ :tEQL, "=", EXPR_BEG,
1328
+ :tIDENTIFIER, "arg", EXPR_ARG)
1434
1329
  end
1435
1330
 
1436
1331
  def test_yylex_identifier_equals_or
@@ -1442,12 +1337,12 @@ class TestRubyLexer < Minitest::Test
1442
1337
  end
1443
1338
 
1444
1339
  def test_yylex_identifier_equals_tilde
1445
- self.lex_state = :expr_fname # can only set via parser's defs
1340
+ self.lex_state = EXPR_FNAME # can only set via parser's defs
1446
1341
 
1447
1342
  assert_lex3("identifier=~",
1448
1343
  nil,
1449
- :tIDENTIFIER, "identifier", :expr_endfn,
1450
- :tMATCH, "=~", :expr_beg)
1344
+ :tIDENTIFIER, "identifier", EXPR_ENDFN,
1345
+ :tMATCH, "=~", EXPR_BEG)
1451
1346
  end
1452
1347
 
1453
1348
  def test_yylex_identifier_gt
@@ -1475,11 +1370,11 @@ class TestRubyLexer < Minitest::Test
1475
1370
  end
1476
1371
 
1477
1372
  def test_yylex_integer
1478
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1373
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1479
1374
  end
1480
1375
 
1481
1376
  def test_yylex_integer_bin
1482
- assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
1377
+ assert_lex3("0b101010", nil, :tINTEGER, 42, EXPR_NUM)
1483
1378
  end
1484
1379
 
1485
1380
  def test_yylex_integer_bin_bad_none
@@ -1491,7 +1386,7 @@ class TestRubyLexer < Minitest::Test
1491
1386
  end
1492
1387
 
1493
1388
  def test_yylex_integer_dec
1494
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1389
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1495
1390
  end
1496
1391
 
1497
1392
  def test_yylex_integer_dec_bad_underscores
@@ -1499,7 +1394,7 @@ class TestRubyLexer < Minitest::Test
1499
1394
  end
1500
1395
 
1501
1396
  def test_yylex_integer_dec_d
1502
- assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
1397
+ assert_lex3("0d42", nil, :tINTEGER, 42, EXPR_NUM)
1503
1398
  end
1504
1399
 
1505
1400
  def test_yylex_integer_dec_d_bad_none
@@ -1510,27 +1405,8 @@ class TestRubyLexer < Minitest::Test
1510
1405
  refute_lex "0d42__24"
1511
1406
  end
1512
1407
 
1513
- def test_yylex_integer_if_modifier
1514
- assert_lex3("123if",
1515
- nil,
1516
- :tINTEGER, 123, :expr_end,
1517
- :kIF_MOD, "if", :expr_beg)
1518
- end
1519
-
1520
- def test_yylex_question_eh_a__20
1521
- setup_lexer_class RubyParser::V20
1522
-
1523
- assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
1524
- end
1525
-
1526
- def test_yylex_question_eh_escape_M_escape_C__20
1527
- setup_lexer_class RubyParser::V20
1528
-
1529
- assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
1530
- end
1531
-
1532
1408
  def test_yylex_integer_hex
1533
- assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
1409
+ assert_lex3 "0x2a", nil, :tINTEGER, 42, EXPR_NUM
1534
1410
  end
1535
1411
 
1536
1412
  def test_yylex_integer_hex_bad_none
@@ -1541,24 +1417,19 @@ class TestRubyLexer < Minitest::Test
1541
1417
  refute_lex "0xab__cd"
1542
1418
  end
1543
1419
 
1544
- def test_yylex_integer_oct
1545
- assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
1546
- end
1547
-
1548
- def test_yylex_integer_oct_bad_range
1549
- refute_lex "08"
1550
- end
1551
-
1552
- def test_yylex_integer_oct_bad_range2
1553
- refute_lex "08"
1420
+ def test_yylex_integer_if_modifier
1421
+ assert_lex3("123if",
1422
+ nil,
1423
+ :tINTEGER, 123, EXPR_NUM,
1424
+ :kIF_MOD, "if", EXPR_PAR)
1554
1425
  end
1555
1426
 
1556
- def test_yylex_integer_oct_bad_underscores
1557
- refute_lex "01__23"
1427
+ def test_yylex_integer_oct
1428
+ assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
1558
1429
  end
1559
1430
 
1560
1431
  def test_yylex_integer_oct_O
1561
- assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
1432
+ assert_lex3 "0O52", nil, :tINTEGER, 42, EXPR_NUM
1562
1433
  end
1563
1434
 
1564
1435
  def test_yylex_integer_oct_O_bad_range
@@ -1570,11 +1441,23 @@ class TestRubyLexer < Minitest::Test
1570
1441
  end
1571
1442
 
1572
1443
  def test_yylex_integer_oct_O_not_bad_none
1573
- assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
1444
+ assert_lex3 "0O ", nil, :tINTEGER, 0, EXPR_NUM
1445
+ end
1446
+
1447
+ def test_yylex_integer_oct_bad_range
1448
+ refute_lex "08"
1449
+ end
1450
+
1451
+ def test_yylex_integer_oct_bad_range2
1452
+ refute_lex "08"
1453
+ end
1454
+
1455
+ def test_yylex_integer_oct_bad_underscores
1456
+ refute_lex "01__23"
1574
1457
  end
1575
1458
 
1576
1459
  def test_yylex_integer_oct_o
1577
- assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
1460
+ assert_lex3 "0o52", nil, :tINTEGER, 42, EXPR_NUM
1578
1461
  end
1579
1462
 
1580
1463
  def test_yylex_integer_oct_o_bad_range
@@ -1586,19 +1469,19 @@ class TestRubyLexer < Minitest::Test
1586
1469
  end
1587
1470
 
1588
1471
  def test_yylex_integer_oct_o_not_bad_none
1589
- assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
1472
+ assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
1590
1473
  end
1591
1474
 
1592
1475
  def test_yylex_integer_trailing
1593
1476
  assert_lex3("1.to_s",
1594
1477
  nil,
1595
- :tINTEGER, 1, :expr_end,
1596
- :tDOT, ".", :expr_dot,
1597
- :tIDENTIFIER, "to_s", :expr_arg)
1478
+ :tINTEGER, 1, EXPR_NUM,
1479
+ :tDOT, ".", EXPR_DOT,
1480
+ :tIDENTIFIER, "to_s", EXPR_ARG)
1598
1481
  end
1599
1482
 
1600
1483
  def test_yylex_integer_underscore
1601
- assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
1484
+ assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
1602
1485
  end
1603
1486
 
1604
1487
  def test_yylex_integer_underscore_bad
@@ -1606,11 +1489,62 @@ class TestRubyLexer < Minitest::Test
1606
1489
  end
1607
1490
 
1608
1491
  def test_yylex_integer_zero
1609
- assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
1492
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1493
+ end
1494
+
1495
+ def test_yylex_is_your_spacebar_broken?
1496
+ assert_lex3(":a!=:b",
1497
+ nil,
1498
+ :tSYMBOL, "a", EXPR_LIT,
1499
+ :tNEQ, "!=", EXPR_BEG,
1500
+ :tSYMBOL, "b", EXPR_LIT)
1501
+ end
1502
+
1503
+ def test_yylex_iter_array_curly
1504
+ # this will lex, but doesn't parse... don't freak out.
1505
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1506
+ nil,
1507
+
1508
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1509
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1510
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1511
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1512
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1513
+ :tRBRACK, "]", EXPR_END, 0, 0,
1514
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1515
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1516
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1517
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1518
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1519
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1520
+ :tRCURLY, "}", EXPR_END, 0, 0)
1521
+ end
1522
+
1523
+ def test_yylex_iter_array_curly__24
1524
+ setup_lexer_class RubyParser::V24
1525
+
1526
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1527
+ s(:iter,
1528
+ s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
1529
+ s(:args, :c, :d)),
1530
+
1531
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1532
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1533
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1534
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1535
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1536
+ :tRBRACK, "]", EXPR_ENDARG, 0, 0,
1537
+ :tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
1538
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1539
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1540
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1541
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1542
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1543
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1610
1544
  end
1611
1545
 
1612
1546
  def test_yylex_ivar
1613
- assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
1547
+ assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
1614
1548
  end
1615
1549
 
1616
1550
  def test_yylex_ivar_bad
@@ -1618,291 +1552,590 @@ class TestRubyLexer < Minitest::Test
1618
1552
  end
1619
1553
 
1620
1554
  def test_yylex_ivar_bad_0_length
1621
- refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
1555
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
1622
1556
  end
1623
1557
 
1624
1558
  def test_yylex_keyword_expr
1625
- self.lex_state = :expr_endarg
1559
+ self.lex_state = EXPR_ENDARG
1560
+
1561
+ assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
1562
+ end
1626
1563
 
1627
- assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
1564
+ def test_yylex_label
1565
+ assert_lex3("{a:",
1566
+ nil,
1567
+ :tLBRACE, "{", EXPR_PAR,
1568
+ :tLABEL, "a", EXPR_LAB)
1569
+ end
1570
+
1571
+ def test_yylex_label_in_params
1572
+ assert_lex3("foo(a:",
1573
+ nil,
1574
+ :tIDENTIFIER, "foo", EXPR_CMDARG,
1575
+ :tLPAREN2, "(", EXPR_PAR,
1576
+ :tLABEL, "a", EXPR_LAB)
1577
+ end
1578
+
1579
+ def test_yylex_lambda_args
1580
+ assert_lex("-> (a) { }",
1581
+ s(:iter, s(:lambda),
1582
+ s(:args, :a)),
1583
+
1584
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1585
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1586
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1587
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1588
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1589
+ :tRCURLY, "}", EXPR_END, 0, 0)
1590
+ end
1591
+
1592
+ def test_yylex_lambda_args__24
1593
+ setup_lexer_class RubyParser::V24
1594
+
1595
+ assert_lex("-> (a) { }",
1596
+ s(:iter, s(:lambda),
1597
+ s(:args, :a)),
1598
+
1599
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1600
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1601
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1602
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1603
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1604
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1605
+ end
1606
+
1607
+ def test_yylex_lambda_args_opt
1608
+ assert_lex("-> (a=nil) { }",
1609
+ s(:iter, s(:lambda),
1610
+ s(:args, s(:lasgn, :a, s(:nil)))),
1611
+
1612
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1613
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1614
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1615
+ :tEQL, "=", EXPR_BEG, 1, 0,
1616
+ :kNIL, "nil", EXPR_END, 1, 0,
1617
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1618
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1619
+ :tRCURLY, "}", EXPR_END, 0, 0)
1620
+ end
1621
+
1622
+ def test_yylex_lambda_args_opt__24
1623
+ setup_lexer_class RubyParser::V24
1624
+
1625
+ assert_lex("-> (a=nil) { }",
1626
+ s(:iter, s(:lambda),
1627
+ s(:args, s(:lasgn, :a, s(:nil)))),
1628
+
1629
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1630
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1631
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1632
+ :tEQL, "=", EXPR_BEG, 1, 0,
1633
+ :kNIL, "nil", EXPR_END, 1, 0,
1634
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1635
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1636
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1637
+ end
1638
+
1639
+ def test_yylex_lambda_as_args_with_block
1640
+ assert_lex3("a -> do end do end",
1641
+ nil,
1642
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1643
+ :tLAMBDA, nil, EXPR_ENDFN,
1644
+ :kDO, "do", EXPR_BEG,
1645
+ :kEND, "end", EXPR_END,
1646
+ :kDO, "do", EXPR_BEG,
1647
+ :kEND, "end", EXPR_END)
1648
+ end
1649
+
1650
+ def test_yylex_lambda_hash
1651
+ assert_lex("-> (a={}) { }",
1652
+ s(:iter, s(:lambda),
1653
+ s(:args, s(:lasgn, :a, s(:hash)))),
1654
+
1655
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1656
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1657
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1658
+ :tEQL, "=", EXPR_BEG, 1, 0,
1659
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1660
+ :tRCURLY, "}", EXPR_END, 1, 0,
1661
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1662
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1663
+ :tRCURLY, "}", EXPR_END, 0, 0)
1664
+ end
1665
+
1666
+ def test_yylex_lambda_hash__24
1667
+ setup_lexer_class RubyParser::V24
1668
+
1669
+ assert_lex("-> (a={}) { }",
1670
+ s(:iter, s(:lambda),
1671
+ s(:args, s(:lasgn, :a, s(:hash)))),
1672
+
1673
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1674
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1675
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1676
+ :tEQL, "=", EXPR_BEG, 1, 0,
1677
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1678
+ :tRCURLY, "}", EXPR_ENDARG, 1, 0,
1679
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1680
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1681
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1682
+ end
1683
+
1684
+ def test_yylex_lasgn_call_same_name
1685
+ assert_lex("a = b.c :d => 1",
1686
+ s(:lasgn, :a,
1687
+ s(:call, s(:call, nil, :b), :c,
1688
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1689
+
1690
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1691
+ :tEQL, "=", EXPR_BEG, 0, 0,
1692
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1693
+ :tDOT, ".", EXPR_DOT, 0, 0,
1694
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
1695
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1696
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1697
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1698
+
1699
+ assert_lex("a = b.a :d => 1",
1700
+ s(:lasgn, :a,
1701
+ s(:call, s(:call, nil, :b), :a,
1702
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1703
+
1704
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1705
+ :tEQL, "=", EXPR_BEG, 0, 0,
1706
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1707
+ :tDOT, ".", EXPR_DOT, 0, 0,
1708
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
1709
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1710
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1711
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1628
1712
  end
1629
1713
 
1630
1714
  def test_yylex_lt
1631
- assert_lex3("<", nil, :tLT, "<", :expr_beg)
1715
+ assert_lex3("<", nil, :tLT, "<", EXPR_BEG)
1632
1716
  end
1633
1717
 
1634
1718
  def test_yylex_lt2
1635
1719
  assert_lex3("a << b",
1636
1720
  nil,
1637
- :tIDENTIFIER, "a", :expr_cmdarg,
1638
- :tLSHFT, "<<", :expr_beg,
1639
- :tIDENTIFIER, "b", :expr_arg)
1721
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1722
+ :tLSHFT, "<<", EXPR_BEG,
1723
+ :tIDENTIFIER, "b", EXPR_ARG)
1640
1724
  end
1641
1725
 
1642
1726
  def test_yylex_lt2_equals
1643
1727
  assert_lex3("a <<= b",
1644
1728
  nil,
1645
- :tIDENTIFIER, "a", :expr_cmdarg,
1646
- :tOP_ASGN, "<<", :expr_beg,
1647
- :tIDENTIFIER, "b", :expr_arg)
1729
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1730
+ :tOP_ASGN, "<<", EXPR_BEG,
1731
+ :tIDENTIFIER, "b", EXPR_ARG)
1648
1732
  end
1649
1733
 
1650
1734
  def test_yylex_lt_equals
1651
- assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
1735
+ assert_lex3("<=", nil, :tLEQ, "<=", EXPR_BEG)
1736
+ end
1737
+
1738
+ def test_yylex_method_parens_chevron
1739
+ assert_lex("a()<<1",
1740
+ s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
1741
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1742
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1743
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1744
+ :tLSHFT, "<<", EXPR_BEG, 0, 0,
1745
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1652
1746
  end
1653
1747
 
1654
1748
  def test_yylex_minus
1655
1749
  assert_lex3("1 - 2",
1656
1750
  nil,
1657
- :tINTEGER, 1, :expr_end,
1658
- :tMINUS, "-", :expr_beg,
1659
- :tINTEGER, 2, :expr_end)
1751
+ :tINTEGER, 1, EXPR_NUM,
1752
+ :tMINUS, "-", EXPR_BEG,
1753
+ :tINTEGER, 2, EXPR_NUM)
1660
1754
  end
1661
1755
 
1662
1756
  def test_yylex_minus_equals
1663
- assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
1757
+ assert_lex3("-=", nil, :tOP_ASGN, "-", EXPR_BEG)
1664
1758
  end
1665
1759
 
1666
1760
  def test_yylex_minus_method
1667
- self.lex_state = :expr_fname
1761
+ self.lex_state = EXPR_FNAME
1668
1762
 
1669
- assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
1763
+ assert_lex3("-", nil, :tMINUS, "-", EXPR_ARG)
1670
1764
  end
1671
1765
 
1672
1766
  def test_yylex_minus_unary_method
1673
- self.lex_state = :expr_fname
1767
+ self.lex_state = EXPR_FNAME
1674
1768
 
1675
- assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
1769
+ assert_lex3("-@", nil, :tUMINUS, "-@", EXPR_ARG)
1676
1770
  end
1677
1771
 
1678
1772
  def test_yylex_minus_unary_number
1679
1773
  assert_lex3("-42",
1680
1774
  nil,
1681
- :tUMINUS_NUM, "-", :expr_beg,
1682
- :tINTEGER, 42, :expr_end)
1775
+ :tUMINUS_NUM, "-", EXPR_BEG,
1776
+ :tINTEGER, 42, EXPR_NUM)
1777
+ end
1778
+
1779
+ def test_yylex_not_at_defn
1780
+ assert_lex("def +@; end",
1781
+ s(:defn, :+@, s(:args), s(:nil)),
1782
+
1783
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1784
+ :tUPLUS, "+@", EXPR_ARG, 0, 0,
1785
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1786
+ :kEND, "end", EXPR_END, 0, 0)
1787
+
1788
+ assert_lex("def !@; end",
1789
+ s(:defn, :"!@", s(:args), s(:nil)),
1790
+
1791
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1792
+ :tUBANG, "!@", EXPR_ARG, 0, 0,
1793
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1794
+ :kEND, "end", EXPR_END, 0, 0)
1795
+ end
1796
+
1797
+ def test_yylex_not_at_ivar
1798
+ assert_lex("!@ivar",
1799
+ s(:call, s(:ivar, :@ivar), :"!"),
1800
+
1801
+ :tBANG, "!", EXPR_BEG, 0, 0,
1802
+ :tIVAR, "@ivar", EXPR_END, 0, 0)
1803
+ end
1804
+
1805
+ def test_yylex_not_unary_method
1806
+ self.lex_state = EXPR_FNAME
1807
+
1808
+ assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
1683
1809
  end
1684
1810
 
1685
1811
  def test_yylex_nth_ref
1686
1812
  assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
1687
1813
  nil,
1688
- :tLBRACK, "[", :expr_beg,
1689
- :tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1690
- :tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1691
- :tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
1692
- :tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
1693
- :tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
1694
- :tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
1695
- :tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
1696
- :tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
1697
- :tNTH_REF, 9, :expr_end,
1698
- :tRBRACK, "]", :expr_endarg)
1814
+ :tLBRACK, "[", EXPR_PAR,
1815
+ :tNTH_REF, 1, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1816
+ :tNTH_REF, 2, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1817
+ :tNTH_REF, 3, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1818
+ :tNTH_REF, 4, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1819
+ :tNTH_REF, 5, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1820
+ :tNTH_REF, 6, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1821
+ :tNTH_REF, 7, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1822
+ :tNTH_REF, 8, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1823
+ :tNTH_REF, 9, EXPR_END,
1824
+ :tRBRACK, "]", EXPR_END)
1825
+ end
1826
+
1827
+ def test_yylex_number_times_ident_times_return_number
1828
+ assert_lex("1 * b * 3",
1829
+ s(:call,
1830
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1831
+ :*, s(:lit, 3)),
1832
+
1833
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1834
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1835
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1836
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1837
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1838
+
1839
+ assert_lex("1 * b *\n 3",
1840
+ s(:call,
1841
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1842
+ :*, s(:lit, 3)),
1843
+
1844
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1845
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1846
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1847
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1848
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1849
+ end
1850
+
1851
+ def test_yylex_numbers
1852
+ assert_lex3 "0b10", nil, :tINTEGER, 2, EXPR_NUM
1853
+ assert_lex3 "0B10", nil, :tINTEGER, 2, EXPR_NUM
1854
+
1855
+ assert_lex3 "0d10", nil, :tINTEGER, 10, EXPR_NUM
1856
+ assert_lex3 "0D10", nil, :tINTEGER, 10, EXPR_NUM
1857
+
1858
+ assert_lex3 "0x10", nil, :tINTEGER, 16, EXPR_NUM
1859
+ assert_lex3 "0X10", nil, :tINTEGER, 16, EXPR_NUM
1860
+
1861
+ assert_lex3 "0o10", nil, :tINTEGER, 8, EXPR_NUM
1862
+ assert_lex3 "0O10", nil, :tINTEGER, 8, EXPR_NUM
1863
+
1864
+ assert_lex3 "0o", nil, :tINTEGER, 0, EXPR_NUM
1865
+ assert_lex3 "0O", nil, :tINTEGER, 0, EXPR_NUM
1866
+
1867
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1868
+
1869
+ refute_lex "0x"
1870
+ refute_lex "0X"
1871
+ refute_lex "0b"
1872
+ refute_lex "0B"
1873
+ refute_lex "0d"
1874
+ refute_lex "0D"
1875
+
1876
+ refute_lex "08"
1877
+ refute_lex "09"
1878
+ refute_lex "0o8"
1879
+ refute_lex "0o9"
1880
+ refute_lex "0O8"
1881
+ refute_lex "0O9"
1882
+
1883
+ refute_lex "1_e1"
1884
+ refute_lex "1_.1"
1885
+ refute_lex "1__1"
1699
1886
  end
1700
1887
 
1701
1888
  def test_yylex_open_bracket
1702
- assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
1889
+ assert_lex3("(", nil, :tLPAREN, "(", EXPR_PAR)
1703
1890
  end
1704
1891
 
1705
1892
  def test_yylex_open_bracket_cmdarg
1706
- self.lex_state = :expr_cmdarg
1893
+ self.lex_state = EXPR_CMDARG
1707
1894
 
1708
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1895
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1709
1896
  end
1710
1897
 
1711
1898
  def test_yylex_open_bracket_exprarg__20
1712
1899
  setup_lexer_class RubyParser::V20
1713
- self.lex_state = :expr_arg
1900
+ self.lex_state = EXPR_ARG
1714
1901
 
1715
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1902
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1716
1903
  end
1717
1904
 
1718
1905
  def test_yylex_open_curly_bracket
1719
- assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
1906
+ assert_lex3("{", nil, :tLBRACE, "{", EXPR_PAR)
1720
1907
  end
1721
1908
 
1722
1909
  def test_yylex_open_curly_bracket_arg
1723
- self.lex_state = :expr_arg
1910
+ self.lex_state = EXPR_ARG
1724
1911
 
1725
1912
  assert_lex3("m { 3 }",
1726
1913
  nil,
1727
- :tIDENTIFIER, "m", :expr_cmdarg,
1728
- :tLCURLY, "{", :expr_beg,
1729
- :tINTEGER, 3, :expr_end,
1730
- :tRCURLY, "}", :expr_endarg)
1914
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1915
+ :tLCURLY, "{", EXPR_PAR,
1916
+ :tINTEGER, 3, EXPR_NUM,
1917
+ :tRCURLY, "}", EXPR_END)
1731
1918
  end
1732
1919
 
1733
1920
  def test_yylex_open_curly_bracket_block
1734
- self.lex_state = :expr_endarg # seen m(3)
1921
+ self.lex_state = EXPR_ENDARG # seen m(3)
1735
1922
 
1736
1923
  assert_lex3("{ 4 }",
1737
1924
  nil,
1738
- :tLBRACE_ARG, "{", :expr_beg,
1739
- :tINTEGER, 4, :expr_end,
1740
- :tRCURLY, "}", :expr_endarg)
1925
+ :tLBRACE_ARG, "{", EXPR_BEG,
1926
+ :tINTEGER, 4, EXPR_NUM,
1927
+ :tRCURLY, "}", EXPR_END)
1741
1928
  end
1742
1929
 
1743
1930
  def test_yylex_open_square_bracket_arg
1744
- self.lex_state = :expr_arg
1931
+ self.lex_state = EXPR_ARG
1745
1932
 
1746
1933
  assert_lex3("m [ 3 ]",
1747
1934
  nil,
1748
- :tIDENTIFIER, "m", :expr_cmdarg,
1749
- :tLBRACK, "[", :expr_beg,
1750
- :tINTEGER, 3, :expr_end,
1751
- :tRBRACK, "]", :expr_endarg)
1935
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1936
+ :tLBRACK, "[", EXPR_PAR,
1937
+ :tINTEGER, 3, EXPR_NUM,
1938
+ :tRBRACK, "]", EXPR_END)
1752
1939
  end
1753
1940
 
1754
1941
  def test_yylex_open_square_bracket_ary
1755
1942
  assert_lex3("[1, 2, 3]",
1756
1943
  nil,
1757
- :tLBRACK, "[", :expr_beg,
1758
- :tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1759
- :tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1760
- :tINTEGER, 3, :expr_end,
1761
- :tRBRACK, "]", :expr_endarg)
1944
+ :tLBRACK, "[", EXPR_PAR,
1945
+ :tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1946
+ :tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1947
+ :tINTEGER, 3, EXPR_NUM,
1948
+ :tRBRACK, "]", EXPR_END)
1762
1949
  end
1763
1950
 
1764
1951
  def test_yylex_open_square_bracket_meth
1765
1952
  assert_lex3("m[3]",
1766
1953
  nil,
1767
- :tIDENTIFIER, "m", :expr_cmdarg,
1768
- :tLBRACK2, "[", :expr_beg,
1769
- :tINTEGER, 3, :expr_end,
1770
- :tRBRACK, "]", :expr_endarg)
1954
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1955
+ :tLBRACK2, "[", EXPR_PAR,
1956
+ :tINTEGER, 3, EXPR_NUM,
1957
+ :tRBRACK, "]", EXPR_END)
1771
1958
  end
1772
1959
 
1773
1960
  def test_yylex_or
1774
- assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
1961
+ assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
1775
1962
  end
1776
1963
 
1777
1964
  def test_yylex_or2
1778
- assert_lex3("||", nil, :tOROP, "||", :expr_beg)
1965
+ assert_lex3("||", nil, :tOROP, "||", EXPR_BEG)
1779
1966
  end
1780
1967
 
1781
1968
  def test_yylex_or2_equals
1782
- assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
1969
+ assert_lex3("||=", nil, :tOP_ASGN, "||", EXPR_BEG)
1783
1970
  end
1784
1971
 
1785
1972
  def test_yylex_or_equals
1786
- assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
1973
+ assert_lex3("|=", nil, :tOP_ASGN, "|", EXPR_BEG)
1787
1974
  end
1788
1975
 
1789
- def test_yylex_percent
1790
- assert_lex3("a % 2",
1791
- nil,
1792
- :tIDENTIFIER, "a", :expr_cmdarg,
1793
- :tPERCENT, "%", :expr_beg,
1794
- :tINTEGER, 2, :expr_end)
1795
- end
1976
+ def test_yylex_paren_string_interpolated_regexp
1977
+ setup_lexer('%( #{(/abcd/)} )',
1978
+ s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
1796
1979
 
1797
- def test_yylex_percent_equals
1798
- assert_lex3("a %= 2",
1799
- nil,
1800
- :tIDENTIFIER, "a", :expr_cmdarg,
1801
- :tOP_ASGN, "%", :expr_beg,
1802
- :tINTEGER, 2, :expr_end)
1803
- end
1980
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
1981
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1982
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1804
1983
 
1805
- def test_yylex_plus
1806
- assert_lex3("1 + 1", # TODO lex_state?
1807
- nil,
1808
- :tINTEGER, 1, :expr_end,
1809
- :tPLUS, "+", :expr_beg,
1810
- :tINTEGER, 1, :expr_end)
1811
- end
1984
+ emulate_string_interpolation do
1985
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
1986
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
1987
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
1988
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
1989
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
1990
+ end
1812
1991
 
1813
- def test_yylex_plus_equals
1814
- assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
1992
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1993
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1994
+
1995
+ refute_lexeme
1815
1996
  end
1816
1997
 
1817
- def test_yylex_plus_method
1818
- self.lex_state = :expr_fname
1998
+ def test_yylex_paren_string_parens_interpolated
1999
+ setup_lexer('%((#{b}#{d}))',
2000
+ s(:dstr,
2001
+ "(",
2002
+ s(:evstr, s(:call, nil, :b)),
2003
+ s(:evstr, s(:call, nil, :d)),
2004
+ s(:str, ")")))
1819
2005
 
1820
- assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
1821
- end
2006
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2007
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2008
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1822
2009
 
1823
- def test_yylex_plus_unary_method
1824
- self.lex_state = :expr_fname
2010
+ emulate_string_interpolation do
2011
+ assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2012
+ end
1825
2013
 
1826
- assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
1827
- end
2014
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1828
2015
 
1829
- def test_yylex_not_unary_method
1830
- self.lex_state = :expr_fname
2016
+ emulate_string_interpolation do
2017
+ assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
2018
+ end
2019
+
2020
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2021
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1831
2022
 
1832
- assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
2023
+ refute_lexeme
1833
2024
  end
1834
2025
 
1835
- def test_yylex_numbers
1836
- assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
1837
- assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
2026
+ def test_yylex_paren_string_parens_interpolated_regexp
2027
+ setup_lexer('%((#{(/abcd/)}))',
2028
+ s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
1838
2029
 
1839
- assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
1840
- assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
2030
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2031
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
1841
2032
 
1842
- assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
1843
- assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
2033
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1844
2034
 
1845
- assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
1846
- assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
2035
+ emulate_string_interpolation do
2036
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
2037
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
2038
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
2039
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
2040
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
2041
+ end
1847
2042
 
1848
- assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
1849
- assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
2043
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2044
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1850
2045
 
1851
- assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
2046
+ refute_lexeme
2047
+ end
1852
2048
 
1853
- refute_lex "0x"
1854
- refute_lex "0X"
1855
- refute_lex "0b"
1856
- refute_lex "0B"
1857
- refute_lex "0d"
1858
- refute_lex "0D"
2049
+ def test_yylex_percent
2050
+ assert_lex3("a % 2",
2051
+ nil,
2052
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2053
+ :tPERCENT, "%", EXPR_BEG,
2054
+ :tINTEGER, 2, EXPR_NUM)
2055
+ end
1859
2056
 
1860
- refute_lex "08"
1861
- refute_lex "09"
1862
- refute_lex "0o8"
1863
- refute_lex "0o9"
1864
- refute_lex "0O8"
1865
- refute_lex "0O9"
2057
+ def test_yylex_percent_equals
2058
+ assert_lex3("a %= 2",
2059
+ nil,
2060
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2061
+ :tOP_ASGN, "%", EXPR_BEG,
2062
+ :tINTEGER, 2, EXPR_NUM)
2063
+ end
2064
+
2065
+ def test_yylex_plus
2066
+ assert_lex3("1 + 1", # TODO lex_state?
2067
+ nil,
2068
+ :tINTEGER, 1, EXPR_NUM,
2069
+ :tPLUS, "+", EXPR_BEG,
2070
+ :tINTEGER, 1, EXPR_NUM)
2071
+ end
2072
+
2073
+ def test_yylex_plus_equals
2074
+ assert_lex3("+=", nil, :tOP_ASGN, "+", EXPR_BEG)
2075
+ end
2076
+
2077
+ def test_yylex_plus_method
2078
+ self.lex_state = EXPR_FNAME
1866
2079
 
1867
- refute_lex "1_e1"
1868
- refute_lex "1_.1"
1869
- refute_lex "1__1"
2080
+ assert_lex3("+", nil, :tPLUS, "+", EXPR_ARG)
2081
+ end
2082
+
2083
+ def test_yylex_plus_unary_method
2084
+ self.lex_state = EXPR_FNAME
2085
+
2086
+ assert_lex3("+@", nil, :tUPLUS, "+@", EXPR_ARG)
1870
2087
  end
1871
2088
 
1872
2089
  def test_yylex_plus_unary_number
1873
- assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
2090
+ assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
1874
2091
  end
1875
2092
 
1876
2093
  def test_yylex_question_bad_eos
1877
2094
  refute_lex "?"
1878
2095
  end
1879
2096
 
2097
+ def test_yylex_question_eh_a__20
2098
+ setup_lexer_class RubyParser::V20
2099
+
2100
+ assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
2101
+ end
2102
+
2103
+ def test_yylex_question_eh_escape_M_escape_C__20
2104
+ setup_lexer_class RubyParser::V20
2105
+
2106
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
2107
+ end
2108
+
2109
+ def test_yylex_question_control_escape
2110
+ assert_lex3('?\C-\]', nil, :tSTRING, ?\C-\], EXPR_END)
2111
+ end
2112
+
1880
2113
  def test_yylex_question_ws
1881
- assert_lex3("? ", nil, :tEH, "?", :expr_value)
1882
- assert_lex3("?\n", nil, :tEH, "?", :expr_value)
1883
- assert_lex3("?\t", nil, :tEH, "?", :expr_value)
1884
- assert_lex3("?\v", nil, :tEH, "?", :expr_value)
1885
- assert_lex3("?\r", nil, :tEH, "?", :expr_value)
1886
- assert_lex3("?\f", nil, :tEH, "?", :expr_value)
2114
+ assert_lex3("? ", nil, :tEH, "?", EXPR_BEG)
2115
+ assert_lex3("?\n", nil, :tEH, "?", EXPR_BEG)
2116
+ assert_lex3("?\t", nil, :tEH, "?", EXPR_BEG)
2117
+ assert_lex3("?\v", nil, :tEH, "?", EXPR_BEG)
2118
+ assert_lex3("?\r", nil, :tEH, "?", EXPR_BEG)
2119
+ assert_lex3("?\f", nil, :tEH, "?", EXPR_BEG)
1887
2120
  end
1888
2121
 
1889
2122
  def test_yylex_question_ws_backslashed__20
1890
2123
  setup_lexer_class RubyParser::V20
1891
2124
 
1892
- assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
1893
- assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
1894
- assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
1895
- assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
1896
- assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
1897
- assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
2125
+ assert_lex3("?\\ ", nil, :tSTRING, " ", EXPR_END)
2126
+ assert_lex3("?\\n", nil, :tSTRING, "\n", EXPR_END)
2127
+ assert_lex3("?\\t", nil, :tSTRING, "\t", EXPR_END)
2128
+ assert_lex3("?\\v", nil, :tSTRING, "\v", EXPR_END)
2129
+ assert_lex3("?\\r", nil, :tSTRING, "\r", EXPR_END)
2130
+ assert_lex3("?\\f", nil, :tSTRING, "\f", EXPR_END)
1898
2131
  end
1899
2132
 
1900
2133
  def test_yylex_rbracket
1901
- assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
2134
+ assert_lex3("]", nil, :tRBRACK, "]", EXPR_END)
1902
2135
  end
1903
2136
 
1904
2137
  def test_yylex_rcurly
1905
- assert_lex("}", nil, :tRCURLY, "}", :expr_endarg, 0, 1) do
2138
+ assert_lex("}", nil, :tRCURLY, "}", EXPR_END, 0, 1) do
1906
2139
  lexer.brace_nest += 2
1907
2140
  end
1908
2141
  end
@@ -1910,18 +2143,18 @@ class TestRubyLexer < Minitest::Test
1910
2143
  def test_yylex_regexp
1911
2144
  assert_lex3("/regexp/",
1912
2145
  nil,
1913
- :tREGEXP_BEG, "/", :expr_beg,
1914
- :tSTRING_CONTENT, "regexp", :expr_beg,
1915
- :tREGEXP_END, "", :expr_end)
2146
+ :tREGEXP_BEG, "/", EXPR_BEG,
2147
+ :tSTRING_CONTENT, "regexp", EXPR_BEG,
2148
+ :tREGEXP_END, "", EXPR_LIT)
1916
2149
  end
1917
2150
 
1918
2151
  def test_yylex_regexp_ambiguous
1919
2152
  assert_lex3("method /regexp/",
1920
2153
  nil,
1921
- :tIDENTIFIER, "method", :expr_cmdarg,
1922
- :tREGEXP_BEG, "/", :expr_cmdarg,
1923
- :tSTRING_CONTENT, "regexp", :expr_cmdarg,
1924
- :tREGEXP_END, "", :expr_end)
2154
+ :tIDENTIFIER, "method", EXPR_CMDARG,
2155
+ :tREGEXP_BEG, "/", EXPR_CMDARG,
2156
+ :tSTRING_CONTENT, "regexp", EXPR_CMDARG,
2157
+ :tREGEXP_END, "", EXPR_LIT)
1925
2158
  end
1926
2159
 
1927
2160
  def test_yylex_regexp_bad
@@ -1933,25 +2166,25 @@ class TestRubyLexer < Minitest::Test
1933
2166
  def test_yylex_regexp_escape_C
1934
2167
  assert_lex3("/regex\\C-x/",
1935
2168
  nil,
1936
- :tREGEXP_BEG, "/", :expr_beg,
1937
- :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
1938
- :tREGEXP_END, "", :expr_end)
2169
+ :tREGEXP_BEG, "/", EXPR_BEG,
2170
+ :tSTRING_CONTENT, "regex\\C-x", EXPR_BEG,
2171
+ :tREGEXP_END, "", EXPR_LIT)
1939
2172
  end
1940
2173
 
1941
2174
  def test_yylex_regexp_escape_C_M
1942
2175
  assert_lex3("/regex\\C-\\M-x/",
1943
2176
  nil,
1944
- :tREGEXP_BEG, "/", :expr_beg,
1945
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1946
- :tREGEXP_END, "", :expr_end)
2177
+ :tREGEXP_BEG, "/", EXPR_BEG,
2178
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2179
+ :tREGEXP_END, "", EXPR_LIT)
1947
2180
  end
1948
2181
 
1949
2182
  def test_yylex_regexp_escape_C_M_craaaazy
1950
2183
  assert_lex3("/regex\\C-\\\n\\M-x/",
1951
2184
  nil,
1952
- :tREGEXP_BEG, "/", :expr_beg,
1953
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1954
- :tREGEXP_END, "", :expr_end)
2185
+ :tREGEXP_BEG, "/", EXPR_BEG,
2186
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2187
+ :tREGEXP_END, "", EXPR_LIT)
1955
2188
  end
1956
2189
 
1957
2190
  def test_yylex_regexp_escape_C_bad_dash
@@ -1977,17 +2210,17 @@ class TestRubyLexer < Minitest::Test
1977
2210
  def test_yylex_regexp_escape_M
1978
2211
  assert_lex3("/regex\\M-x/",
1979
2212
  nil,
1980
- :tREGEXP_BEG, "/", :expr_beg,
1981
- :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
1982
- :tREGEXP_END, "", :expr_end)
2213
+ :tREGEXP_BEG, "/", EXPR_BEG,
2214
+ :tSTRING_CONTENT, "regex\\M-x", EXPR_BEG,
2215
+ :tREGEXP_END, "", EXPR_LIT)
1983
2216
  end
1984
2217
 
1985
2218
  def test_yylex_regexp_escape_M_C
1986
2219
  assert_lex3("/regex\\M-\\C-x/",
1987
2220
  nil,
1988
- :tREGEXP_BEG, "/", :expr_beg,
1989
- :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
1990
- :tREGEXP_END, "", :expr_end)
2221
+ :tREGEXP_BEG, "/", EXPR_BEG,
2222
+ :tSTRING_CONTENT, "regex\\M-\\C-x", EXPR_BEG,
2223
+ :tREGEXP_END, "", EXPR_LIT)
1991
2224
  end
1992
2225
 
1993
2226
  def test_yylex_regexp_escape_M_bad_dash
@@ -2009,49 +2242,41 @@ class TestRubyLexer < Minitest::Test
2009
2242
  def test_yylex_regexp_escape_backslash_slash
2010
2243
  assert_lex3("/\\//",
2011
2244
  nil,
2012
- :tREGEXP_BEG, "/", :expr_beg,
2013
- :tSTRING_CONTENT, "\\/", :expr_beg,
2014
- :tREGEXP_END, "", :expr_end)
2245
+ :tREGEXP_BEG, "/", EXPR_BEG,
2246
+ :tSTRING_CONTENT, "\\/", EXPR_BEG,
2247
+ :tREGEXP_END, "", EXPR_LIT)
2015
2248
  end
2016
2249
 
2017
2250
  def test_yylex_regexp_escape_backslash_terminator
2018
2251
  assert_lex3("%r%blah\\%blah%",
2019
2252
  nil,
2020
- :tREGEXP_BEG, "%r\000", :expr_beg,
2021
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2022
- :tREGEXP_END, "", :expr_end)
2023
- end
2024
-
2025
- def test_yylex_regexp_escaped_delim
2026
- assert_lex3("%r!blah(?\\!blah)!",
2027
- nil,
2028
- :tREGEXP_BEG, "%r\000", :expr_beg,
2029
- :tSTRING_CONTENT, "blah(?!blah)", :expr_beg,
2030
- :tREGEXP_END, "", :expr_end)
2253
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2254
+ :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2255
+ :tREGEXP_END, "", EXPR_LIT)
2031
2256
  end
2032
2257
 
2033
2258
  def test_yylex_regexp_escape_backslash_terminator_meta1
2034
2259
  assert_lex3("%r{blah\\}blah}",
2035
2260
  nil,
2036
- :tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
2037
- :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
2038
- :tREGEXP_END, "", :expr_end)
2261
+ :tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
2262
+ :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2263
+ :tREGEXP_END, "", EXPR_LIT)
2039
2264
  end
2040
2265
 
2041
2266
  def test_yylex_regexp_escape_backslash_terminator_meta2
2042
2267
  assert_lex3("%r/blah\\/blah/",
2043
2268
  nil,
2044
- :tREGEXP_BEG, "%r\000", :expr_beg,
2045
- :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
2046
- :tREGEXP_END, "", :expr_end)
2269
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2270
+ :tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
2271
+ :tREGEXP_END, "", EXPR_LIT)
2047
2272
  end
2048
2273
 
2049
2274
  def test_yylex_regexp_escape_backslash_terminator_meta3
2050
2275
  assert_lex3("%r/blah\\%blah/",
2051
2276
  nil,
2052
- :tREGEXP_BEG, "%r\000", :expr_beg,
2053
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2054
- :tREGEXP_END, "", :expr_end)
2277
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2278
+ :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2279
+ :tREGEXP_END, "", EXPR_LIT)
2055
2280
  end
2056
2281
 
2057
2282
  def test_yylex_regexp_escape_bad_eos
@@ -2061,50 +2286,50 @@ class TestRubyLexer < Minitest::Test
2061
2286
  def test_yylex_regexp_escape_bs
2062
2287
  assert_lex3("/regex\\\\regex/",
2063
2288
  nil,
2064
- :tREGEXP_BEG, "/", :expr_beg,
2065
- :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
2066
- :tREGEXP_END, "", :expr_end)
2289
+ :tREGEXP_BEG, "/", EXPR_BEG,
2290
+ :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2291
+ :tREGEXP_END, "", EXPR_LIT)
2067
2292
  end
2068
2293
 
2069
2294
  def test_yylex_regexp_escape_c
2070
2295
  assert_lex3("/regex\\cxxx/",
2071
2296
  nil,
2072
- :tREGEXP_BEG, "/", :expr_beg,
2073
- :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
2074
- :tREGEXP_END, "", :expr_end)
2297
+ :tREGEXP_BEG, "/", EXPR_BEG,
2298
+ :tSTRING_CONTENT, "regex\\cxxx", EXPR_BEG,
2299
+ :tREGEXP_END, "", EXPR_LIT)
2075
2300
  end
2076
2301
 
2077
2302
  def test_yylex_regexp_escape_c_backslash
2078
2303
  assert_lex3("/regex\\c\\n/",
2079
2304
  nil,
2080
- :tREGEXP_BEG, "/", :expr_beg,
2081
- :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
2082
- :tREGEXP_END, "", :expr_end)
2305
+ :tREGEXP_BEG, "/", EXPR_BEG,
2306
+ :tSTRING_CONTENT, "regex\\c\\n", EXPR_BEG,
2307
+ :tREGEXP_END, "", EXPR_LIT)
2083
2308
  end
2084
2309
 
2085
2310
  def test_yylex_regexp_escape_chars
2086
2311
  assert_lex3("/re\\tge\\nxp/",
2087
2312
  nil,
2088
- :tREGEXP_BEG, "/", :expr_beg,
2089
- :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
2090
- :tREGEXP_END, "", :expr_end)
2313
+ :tREGEXP_BEG, "/", EXPR_BEG,
2314
+ :tSTRING_CONTENT, "re\\tge\\nxp", EXPR_BEG,
2315
+ :tREGEXP_END, "", EXPR_LIT)
2091
2316
  end
2092
2317
 
2093
2318
  def test_yylex_regexp_escape_double_backslash
2094
2319
  regexp = '/[\\/\\\\]$/'
2095
2320
  assert_lex3(regexp.dup,
2096
2321
  nil,
2097
- :tREGEXP_BEG, "/", :expr_beg,
2098
- :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
2099
- :tREGEXP_END, "", :expr_end)
2322
+ :tREGEXP_BEG, "/", EXPR_BEG,
2323
+ :tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
2324
+ :tREGEXP_END, "", EXPR_LIT)
2100
2325
  end
2101
2326
 
2102
2327
  def test_yylex_regexp_escape_hex
2103
2328
  assert_lex3("/regex\\x61xp/",
2104
2329
  nil,
2105
- :tREGEXP_BEG, "/", :expr_beg,
2106
- :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
2107
- :tREGEXP_END, "", :expr_end)
2330
+ :tREGEXP_BEG, "/", EXPR_BEG,
2331
+ :tSTRING_CONTENT, "regex\\x61xp", EXPR_BEG,
2332
+ :tREGEXP_END, "", EXPR_LIT)
2108
2333
  end
2109
2334
 
2110
2335
  def test_yylex_regexp_escape_hex_bad
@@ -2114,412 +2339,427 @@ class TestRubyLexer < Minitest::Test
2114
2339
  def test_yylex_regexp_escape_hex_one
2115
2340
  assert_lex3("/^[\\xd\\xa]{2}/on",
2116
2341
  nil,
2117
- :tREGEXP_BEG, "/", :expr_beg,
2118
- :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
2119
- :tREGEXP_END, "on", :expr_end)
2342
+ :tREGEXP_BEG, "/", EXPR_BEG,
2343
+ :tSTRING_CONTENT, "^[\\xd\\xa]{2}", EXPR_BEG,
2344
+ :tREGEXP_END, "on", EXPR_LIT)
2120
2345
  end
2121
2346
 
2122
2347
  def test_yylex_regexp_escape_oct1
2123
2348
  assert_lex3("/regex\\0xp/",
2124
2349
  nil,
2125
- :tREGEXP_BEG, "/", :expr_beg,
2126
- :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
2127
- :tREGEXP_END, "", :expr_end)
2350
+ :tREGEXP_BEG, "/", EXPR_BEG,
2351
+ :tSTRING_CONTENT, "regex\\0xp", EXPR_BEG,
2352
+ :tREGEXP_END, "", EXPR_LIT)
2128
2353
  end
2129
2354
 
2130
2355
  def test_yylex_regexp_escape_oct2
2131
2356
  assert_lex3("/regex\\07xp/",
2132
2357
  nil,
2133
- :tREGEXP_BEG, "/", :expr_beg,
2134
- :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
2135
- :tREGEXP_END, "", :expr_end)
2358
+ :tREGEXP_BEG, "/", EXPR_BEG,
2359
+ :tSTRING_CONTENT, "regex\\07xp", EXPR_BEG,
2360
+ :tREGEXP_END, "", EXPR_LIT)
2136
2361
  end
2137
2362
 
2138
2363
  def test_yylex_regexp_escape_oct3
2139
2364
  assert_lex3("/regex\\10142/",
2140
2365
  nil,
2141
- :tREGEXP_BEG, "/", :expr_beg,
2142
- :tSTRING_CONTENT, "regex\\10142", :expr_beg,
2143
- :tREGEXP_END, "", :expr_end)
2366
+ :tREGEXP_BEG, "/", EXPR_BEG,
2367
+ :tSTRING_CONTENT, "regex\\10142", EXPR_BEG,
2368
+ :tREGEXP_END, "", EXPR_LIT)
2144
2369
  end
2145
2370
 
2146
2371
  def test_yylex_regexp_escape_return
2147
2372
  assert_lex3("/regex\\\nregex/",
2148
2373
  nil,
2149
- :tREGEXP_BEG, "/", :expr_beg,
2150
- :tSTRING_CONTENT, "regexregex", :expr_beg,
2151
- :tREGEXP_END, "", :expr_end)
2374
+ :tREGEXP_BEG, "/", EXPR_BEG,
2375
+ :tSTRING_CONTENT, "regexregex", EXPR_BEG,
2376
+ :tREGEXP_END, "", EXPR_LIT)
2377
+ end
2378
+
2379
+ def test_yylex_regexp_escaped_delim
2380
+ assert_lex3("%r!blah(?\\!blah)!",
2381
+ nil,
2382
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2383
+ :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2384
+ :tREGEXP_END, "", EXPR_LIT)
2152
2385
  end
2153
2386
 
2154
2387
  def test_yylex_regexp_nm
2155
2388
  assert_lex3("/.*/nm",
2156
2389
  nil,
2157
- :tREGEXP_BEG, "/", :expr_beg,
2158
- :tSTRING_CONTENT, ".*", :expr_beg,
2159
- :tREGEXP_END, "nm", :expr_end)
2390
+ :tREGEXP_BEG, "/", EXPR_BEG,
2391
+ :tSTRING_CONTENT, ".*", EXPR_BEG,
2392
+ :tREGEXP_END, "nm", EXPR_LIT)
2393
+ end
2394
+
2395
+ def test_yylex_required_kwarg_no_value_22
2396
+ setup_lexer_class RubyParser::V22
2397
+
2398
+ assert_lex3("def foo a:, b:\nend",
2399
+ nil,
2400
+ :kDEF, "def", EXPR_FNAME,
2401
+ :tIDENTIFIER, "foo", EXPR_ENDFN,
2402
+ :tLABEL, "a", EXPR_LAB,
2403
+ :tCOMMA, ",", EXPR_PAR,
2404
+ :tLABEL, "b", EXPR_LAB,
2405
+ :kEND, "end", EXPR_END)
2160
2406
  end
2161
2407
 
2162
2408
  def test_yylex_rparen
2163
- assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
2409
+ assert_lex3(")", nil, :tRPAREN, ")", EXPR_ENDFN)
2164
2410
  end
2165
2411
 
2166
2412
  def test_yylex_rshft
2167
2413
  assert_lex3("a >> 2",
2168
2414
  nil,
2169
- :tIDENTIFIER, "a", :expr_cmdarg,
2170
- :tRSHFT, ">>", :expr_beg,
2171
- :tINTEGER, 2, :expr_end)
2415
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2416
+ :tRSHFT, ">>", EXPR_BEG,
2417
+ :tINTEGER, 2, EXPR_NUM)
2172
2418
  end
2173
2419
 
2174
2420
  def test_yylex_rshft_equals
2175
2421
  assert_lex3("a >>= 2",
2176
2422
  nil,
2177
- :tIDENTIFIER, "a", :expr_cmdarg,
2178
- :tOP_ASGN, ">>", :expr_beg,
2179
- :tINTEGER, 2, :expr_end)
2423
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2424
+ :tOP_ASGN, ">>", EXPR_BEG,
2425
+ :tINTEGER, 2, EXPR_NUM)
2180
2426
  end
2181
2427
 
2182
2428
  def test_yylex_star
2183
2429
  assert_lex3("a * ",
2184
2430
  nil,
2185
- :tIDENTIFIER, "a", :expr_cmdarg,
2186
- :tSTAR2, "*", :expr_beg)
2431
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2432
+ :tSTAR2, "*", EXPR_BEG)
2187
2433
  end
2188
2434
 
2189
2435
  def test_yylex_star2
2190
2436
  assert_lex3("a ** ",
2191
2437
  nil,
2192
- :tIDENTIFIER, "a", :expr_cmdarg,
2193
- :tPOW, "**", :expr_beg)
2438
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2439
+ :tPOW, "**", EXPR_BEG)
2194
2440
  end
2195
2441
 
2196
2442
  def test_yylex_star2_equals
2197
2443
  assert_lex3("a **= ",
2198
2444
  nil,
2199
- :tIDENTIFIER, "a", :expr_cmdarg,
2200
- :tOP_ASGN, "**", :expr_beg)
2445
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2446
+ :tOP_ASGN, "**", EXPR_BEG)
2201
2447
  end
2202
2448
 
2203
2449
  def test_yylex_star_arg
2204
- self.lex_state = :expr_arg
2450
+ self.lex_state = EXPR_ARG
2205
2451
 
2206
2452
  assert_lex3(" *a",
2207
2453
  nil,
2208
- :tSTAR, "*", :expr_beg,
2209
- :tIDENTIFIER, "a", :expr_arg)
2454
+ :tSTAR, "*", EXPR_BEG,
2455
+ :tIDENTIFIER, "a", EXPR_ARG)
2210
2456
  end
2211
2457
 
2212
2458
  def test_yylex_star_arg_beg
2213
- self.lex_state = :expr_beg
2459
+ self.lex_state = EXPR_BEG
2214
2460
 
2215
2461
  assert_lex3("*a",
2216
2462
  nil,
2217
- :tSTAR, "*", :expr_beg,
2218
- :tIDENTIFIER, "a", :expr_arg)
2463
+ :tSTAR, "*", EXPR_BEG,
2464
+ :tIDENTIFIER, "a", EXPR_ARG)
2219
2465
  end
2220
2466
 
2221
2467
  def test_yylex_star_arg_beg_fname
2222
- self.lex_state = :expr_fname
2468
+ self.lex_state = EXPR_FNAME
2223
2469
 
2224
2470
  assert_lex3("*a",
2225
2471
  nil,
2226
- :tSTAR2, "*", :expr_arg,
2227
- :tIDENTIFIER, "a", :expr_arg)
2472
+ :tSTAR2, "*", EXPR_ARG,
2473
+ :tIDENTIFIER, "a", EXPR_ARG)
2228
2474
  end
2229
2475
 
2230
2476
  def test_yylex_star_arg_beg_fname2
2231
- self.lex_state = :expr_fname
2477
+ self.lex_state = EXPR_FNAME
2232
2478
 
2233
2479
  assert_lex3("*a",
2234
2480
  nil,
2235
- :tSTAR2, "*", :expr_arg,
2236
- :tIDENTIFIER, "a", :expr_arg)
2481
+ :tSTAR2, "*", EXPR_ARG,
2482
+ :tIDENTIFIER, "a", EXPR_ARG)
2237
2483
  end
2238
2484
 
2239
2485
  def test_yylex_star_equals
2240
2486
  assert_lex3("a *= ",
2241
2487
  nil,
2242
- :tIDENTIFIER, "a", :expr_cmdarg,
2243
- :tOP_ASGN, "*", :expr_beg)
2488
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2489
+ :tOP_ASGN, "*", EXPR_BEG)
2244
2490
  end
2245
2491
 
2246
2492
  def test_yylex_string_bad_eos
2247
- refute_lex('%', :tSTRING_BEG, '%')
2493
+ refute_lex("%", :tSTRING_BEG, "%")
2248
2494
  end
2249
2495
 
2250
2496
  def test_yylex_string_bad_eos_quote
2251
- refute_lex('%{nest', :tSTRING_BEG, '%}')
2497
+ refute_lex("%{nest", :tSTRING_BEG, "%}")
2252
2498
  end
2253
2499
 
2254
2500
  def test_yylex_string_double
2255
- assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
2501
+ assert_lex3("\"string\"", nil, :tSTRING, "string", EXPR_END)
2256
2502
  end
2257
2503
 
2258
2504
  def test_yylex_string_double_escape_C
2259
- assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
2505
+ assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", EXPR_END)
2260
2506
  end
2261
2507
 
2262
2508
  def test_yylex_string_double_escape_C_backslash
2263
2509
  assert_lex3("\"\\C-\\\\\"",
2264
2510
  nil,
2265
- :tSTRING_BEG, "\"", :expr_beg,
2266
- :tSTRING_CONTENT, "\034", :expr_beg,
2267
- :tSTRING_END, "\"", :expr_end)
2511
+ :tSTRING_BEG, "\"", EXPR_BEG,
2512
+ :tSTRING_CONTENT, "\034", EXPR_BEG,
2513
+ :tSTRING_END, "\"", EXPR_LIT)
2268
2514
  end
2269
2515
 
2270
2516
  def test_yylex_string_double_escape_C_escape
2271
2517
  assert_lex3("\"\\C-\\M-a\"",
2272
2518
  nil,
2273
- :tSTRING_BEG, "\"", :expr_beg,
2274
- :tSTRING_CONTENT, "\201", :expr_beg,
2275
- :tSTRING_END, "\"", :expr_end)
2519
+ :tSTRING_BEG, "\"", EXPR_BEG,
2520
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2521
+ :tSTRING_END, "\"", EXPR_LIT)
2276
2522
  end
2277
2523
 
2278
2524
  def test_yylex_string_double_escape_C_question
2279
- assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
2280
- end
2281
-
2282
- def test_yylex_string_utf8_simple
2283
- chr = [0x3024].pack("U")
2284
-
2285
- assert_lex3('"\u{3024}"',
2286
- s(:str, chr),
2287
- :tSTRING, chr, :expr_end)
2288
- end
2289
-
2290
- def test_yylex_string_utf8_complex
2291
- chr = [0x3024].pack("U")
2292
-
2293
- assert_lex3('"#@a\u{3024}"',
2294
- s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2295
- :tSTRING_BEG, '"', :expr_beg,
2296
- :tSTRING_DVAR, nil, :expr_beg,
2297
- :tSTRING_CONTENT, "@a"+chr, :expr_beg,
2298
- :tSTRING_END, '"', :expr_end)
2525
+ assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", EXPR_END)
2299
2526
  end
2300
2527
 
2301
2528
  def test_yylex_string_double_escape_M
2302
2529
  chr = "\341"
2303
- chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
2304
-
2305
- assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
2306
- end
2307
2530
 
2308
- def test_why_does_ruby_hate_me?
2309
- assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2310
- nil,
2311
- :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
2531
+ assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, EXPR_END)
2312
2532
  end
2313
2533
 
2314
2534
  def test_yylex_string_double_escape_M_backslash
2315
2535
  assert_lex3("\"\\M-\\\\\"",
2316
2536
  nil,
2317
- :tSTRING_BEG, "\"", :expr_beg,
2318
- :tSTRING_CONTENT, "\334", :expr_beg,
2319
- :tSTRING_END, "\"", :expr_end)
2537
+ :tSTRING_BEG, "\"", EXPR_BEG,
2538
+ :tSTRING_CONTENT, "\334", EXPR_BEG,
2539
+ :tSTRING_END, "\"", EXPR_LIT)
2320
2540
  end
2321
2541
 
2322
2542
  def test_yylex_string_double_escape_M_escape
2323
2543
  assert_lex3("\"\\M-\\C-a\"",
2324
2544
  nil,
2325
- :tSTRING_BEG, "\"", :expr_beg,
2326
- :tSTRING_CONTENT, "\201", :expr_beg,
2327
- :tSTRING_END, "\"", :expr_end)
2545
+ :tSTRING_BEG, "\"", EXPR_BEG,
2546
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2547
+ :tSTRING_END, "\"", EXPR_LIT)
2328
2548
  end
2329
2549
 
2330
2550
  def test_yylex_string_double_escape_bs1
2331
- assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
2551
+ assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", EXPR_END)
2332
2552
  end
2333
2553
 
2334
2554
  def test_yylex_string_double_escape_bs2
2335
- assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
2555
+ assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", EXPR_END)
2336
2556
  end
2337
2557
 
2338
2558
  def test_yylex_string_double_escape_c
2339
- assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
2559
+ assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", EXPR_END)
2340
2560
  end
2341
2561
 
2342
2562
  def test_yylex_string_double_escape_c_backslash
2343
- assert_lex3("\"\\c\\\"",
2344
- nil,
2345
- :tSTRING_BEG, "\"", :expr_beg,
2346
- :tSTRING_CONTENT, "\034", :expr_beg,
2347
- :tSTRING_END, "\"", :expr_end)
2563
+ refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
2348
2564
  end
2349
2565
 
2350
2566
  def test_yylex_string_double_escape_c_escape
2351
2567
  assert_lex3("\"\\c\\M-a\"",
2352
2568
  nil,
2353
- :tSTRING_BEG, "\"", :expr_beg,
2354
- :tSTRING_CONTENT, "\201", :expr_beg,
2355
- :tSTRING_END, "\"", :expr_end)
2569
+ :tSTRING_BEG, "\"", EXPR_BEG,
2570
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2571
+ :tSTRING_END, "\"", EXPR_LIT)
2356
2572
  end
2357
2573
 
2358
2574
  def test_yylex_string_double_escape_c_question
2359
- assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
2575
+ assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", EXPR_END)
2360
2576
  end
2361
2577
 
2362
2578
  def test_yylex_string_double_escape_chars
2363
- assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
2579
+ assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", EXPR_END)
2364
2580
  end
2365
2581
 
2366
2582
  def test_yylex_string_double_escape_hex
2367
- assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
2583
+ assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", EXPR_END)
2368
2584
  end
2369
2585
 
2370
2586
  def test_yylex_string_double_escape_octal
2371
- assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
2587
+ assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", EXPR_END)
2372
2588
  end
2373
2589
 
2374
2590
  def test_yylex_string_double_escape_octal_fucked
2375
- assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
2591
+ assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", EXPR_END)
2376
2592
  end
2377
2593
 
2378
2594
  def test_yylex_string_double_interp
2379
2595
  assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
2380
2596
  nil,
2381
- :tSTRING_BEG, "\"", :expr_beg,
2382
- :tSTRING_CONTENT, "blah #x a ", :expr_beg,
2383
- :tSTRING_DVAR, nil, :expr_beg,
2384
- :tSTRING_CONTENT, "@a b ", :expr_beg,
2385
- :tSTRING_DVAR, nil, :expr_beg,
2386
- :tSTRING_CONTENT, "$b c ", :expr_beg,
2387
- :tSTRING_DBEG, nil, :expr_beg,
2388
- :tSTRING_CONTENT, "3} # ", :expr_beg,
2389
- :tSTRING_END, "\"", :expr_end)
2390
- end
2391
-
2392
- def test_yylex_string_double_pound_dollar_bad
2393
- assert_lex3('"#$%"', nil,
2394
-
2395
- :tSTRING_BEG, "\"", :expr_beg,
2396
- :tSTRING_CONTENT, '#$%', :expr_beg,
2397
- :tSTRING_END, "\"", :expr_end)
2597
+ :tSTRING_BEG, "\"", EXPR_BEG,
2598
+ :tSTRING_CONTENT, "blah #x a ", EXPR_BEG,
2599
+ :tSTRING_DVAR, nil, EXPR_BEG,
2600
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG,
2601
+ :tSTRING_DVAR, nil, EXPR_BEG,
2602
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2603
+ :tSTRING_DBEG, nil, EXPR_BEG,
2604
+ :tSTRING_CONTENT, "3} # ", EXPR_BEG,
2605
+ :tSTRING_END, "\"", EXPR_LIT)
2398
2606
  end
2399
2607
 
2400
2608
  def test_yylex_string_double_nested_curlies
2401
2609
  assert_lex3("%{nest{one{two}one}nest}",
2402
2610
  nil,
2403
- :tSTRING_BEG, "%}", :expr_beg,
2404
- :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
2405
- :tSTRING_END, "}", :expr_end)
2611
+ :tSTRING_BEG, "%}", EXPR_BEG,
2612
+ :tSTRING_CONTENT, "nest{one{two}one}nest", EXPR_BEG,
2613
+ :tSTRING_END, "}", EXPR_LIT)
2406
2614
  end
2407
2615
 
2408
2616
  def test_yylex_string_double_no_interp
2409
- assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
2410
- assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
2617
+ assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", EXPR_END)
2618
+ assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", EXPR_END)
2411
2619
  end
2412
2620
 
2413
- def test_yylex_string_escape_x_single
2414
- assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
2621
+ def test_yylex_string_double_pound_dollar_bad
2622
+ assert_lex3('"#$%"', nil,
2623
+
2624
+ :tSTRING_BEG, "\"", EXPR_BEG,
2625
+ :tSTRING_CONTENT, "#\$%", EXPR_BEG,
2626
+ :tSTRING_END, "\"", EXPR_LIT)
2415
2627
  end
2416
2628
 
2417
- def test_yylex_string_pct_i
2418
- assert_lex3("%i[s1 s2\ns3]",
2419
- nil,
2420
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2421
- :tSTRING_CONTENT, "s1", :expr_beg,
2422
- :tSPACE, nil, :expr_beg,
2423
- :tSTRING_CONTENT, "s2", :expr_beg,
2424
- :tSPACE, nil, :expr_beg,
2425
- :tSTRING_CONTENT, "s3", :expr_beg,
2426
- :tSPACE, nil, :expr_beg,
2427
- :tSTRING_END, nil, :expr_end)
2629
+ def test_yylex_string_escape_x_single
2630
+ assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", EXPR_END)
2428
2631
  end
2429
2632
 
2430
2633
  def test_yylex_string_pct_I
2431
2634
  assert_lex3("%I[s1 s2\ns3]",
2432
2635
  nil,
2433
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2434
- :tSTRING_CONTENT, "s1", :expr_beg,
2435
- :tSPACE, nil, :expr_beg,
2436
- :tSTRING_CONTENT, "s2", :expr_beg,
2437
- :tSPACE, nil, :expr_beg,
2438
- :tSTRING_CONTENT, "s3", :expr_beg,
2439
- :tSPACE, nil, :expr_beg,
2440
- :tSTRING_END, nil, :expr_end)
2636
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2637
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2638
+ :tSPACE, nil, EXPR_BEG,
2639
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2640
+ :tSPACE, nil, EXPR_BEG,
2641
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2642
+ :tSPACE, nil, EXPR_BEG,
2643
+ :tSTRING_END, nil, EXPR_LIT)
2441
2644
  end
2442
2645
 
2443
- def test_yylex_string_pct_i_extra_space
2444
- assert_lex3("%i[ s1 s2\ns3 ]",
2646
+ def test_yylex_string_pct_I_extra_space
2647
+ assert_lex3("%I[ s1 s2\ns3 ]",
2445
2648
  nil,
2446
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2447
- :tSTRING_CONTENT, "s1", :expr_beg,
2448
- :tSPACE, nil, :expr_beg,
2449
- :tSTRING_CONTENT, "s2", :expr_beg,
2450
- :tSPACE, nil, :expr_beg,
2451
- :tSTRING_CONTENT, "s3", :expr_beg,
2452
- :tSPACE, nil, :expr_beg,
2453
- :tSTRING_END, nil, :expr_end)
2649
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2650
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2651
+ :tSPACE, nil, EXPR_BEG,
2652
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2653
+ :tSPACE, nil, EXPR_BEG,
2654
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2655
+ :tSPACE, nil, EXPR_BEG,
2656
+ :tSTRING_END, nil, EXPR_LIT)
2454
2657
  end
2455
2658
 
2456
- def test_yylex_string_pct_I_extra_space
2457
- assert_lex3("%I[ s1 s2\ns3 ]",
2659
+ def test_yylex_string_pct_Q
2660
+ assert_lex3("%Q[s1 s2]",
2458
2661
  nil,
2459
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2460
- :tSTRING_CONTENT, "s1", :expr_beg,
2461
- :tSPACE, nil, :expr_beg,
2462
- :tSTRING_CONTENT, "s2", :expr_beg,
2463
- :tSPACE, nil, :expr_beg,
2464
- :tSTRING_CONTENT, "s3", :expr_beg,
2465
- :tSPACE, nil, :expr_beg,
2466
- :tSTRING_END, nil, :expr_end)
2662
+ :tSTRING_BEG, "%Q[", EXPR_BEG,
2663
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2664
+ :tSTRING_END, "]", EXPR_LIT)
2467
2665
  end
2468
2666
 
2469
- def test_yylex_string_pct_q
2470
- assert_lex3("%q[s1 s2]",
2667
+ def test_yylex_string_pct_Q_null_wtf?
2668
+ assert_lex3("%Q\0s1 s2\0",
2471
2669
  nil,
2472
- :tSTRING_BEG, "%q[", :expr_beg,
2473
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2474
- :tSTRING_END, "]", :expr_end)
2670
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2671
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2672
+ :tSTRING_END, "\0", EXPR_LIT)
2475
2673
  end
2476
2674
 
2477
- def test_yylex_string_pct_Q
2478
- assert_lex3("%Q[s1 s2]",
2675
+ def test_yylex_string_pct_Q_bang
2676
+ assert_lex3("%Q!s1 s2!",
2479
2677
  nil,
2480
- :tSTRING_BEG, "%Q[", :expr_beg,
2481
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2482
- :tSTRING_END, "]", :expr_end)
2678
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2679
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2680
+ :tSTRING_END, "!", EXPR_LIT)
2483
2681
  end
2484
2682
 
2485
2683
  def test_yylex_string_pct_W
2486
2684
  assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2487
2685
  nil,
2488
- :tWORDS_BEG, "%W[", :expr_beg,
2489
- :tSTRING_CONTENT, "s1", :expr_beg,
2490
- :tSPACE, nil, :expr_beg,
2491
- :tSTRING_CONTENT, "s2", :expr_beg,
2492
- :tSPACE, nil, :expr_beg,
2493
- :tSTRING_CONTENT, "s3", :expr_beg,
2494
- :tSPACE, nil, :expr_beg,
2495
- :tSTRING_END, nil, :expr_end)
2686
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2687
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2688
+ :tSPACE, nil, EXPR_BEG,
2689
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2690
+ :tSPACE, nil, EXPR_BEG,
2691
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2692
+ :tSPACE, nil, EXPR_BEG,
2693
+ :tSTRING_END, nil, EXPR_LIT)
2496
2694
  end
2497
2695
 
2498
2696
  def test_yylex_string_pct_W_bs_nl
2499
2697
  assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2500
2698
  nil,
2501
- :tWORDS_BEG, "%W[", :expr_beg,
2502
- :tSTRING_CONTENT, "s1", :expr_beg,
2503
- :tSPACE, nil, :expr_beg,
2504
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2505
- :tSPACE, nil, :expr_beg,
2506
- :tSTRING_END, nil, :expr_end)
2699
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2700
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2701
+ :tSPACE, nil, EXPR_BEG,
2702
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2703
+ :tSPACE, nil, EXPR_BEG,
2704
+ :tSTRING_END, nil, EXPR_LIT)
2507
2705
  end
2508
2706
 
2509
2707
  def test_yylex_string_pct_angle
2510
2708
  assert_lex3("%<blah>",
2511
2709
  nil,
2512
- :tSTRING_BEG, "%>", :expr_beg,
2513
- :tSTRING_CONTENT, "blah", :expr_beg,
2514
- :tSTRING_END, ">", :expr_end)
2710
+ :tSTRING_BEG, "%>", EXPR_BEG,
2711
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2712
+ :tSTRING_END, ">", EXPR_LIT)
2713
+ end
2714
+
2715
+ def test_yylex_string_pct_i
2716
+ assert_lex3("%i[s1 s2\ns3]",
2717
+ nil,
2718
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2719
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2720
+ :tSPACE, nil, EXPR_BEG,
2721
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2722
+ :tSPACE, nil, EXPR_BEG,
2723
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2724
+ :tSPACE, nil, EXPR_BEG,
2725
+ :tSTRING_END, nil, EXPR_LIT)
2726
+ end
2727
+
2728
+ def test_yylex_string_pct_i_extra_space
2729
+ assert_lex3("%i[ s1 s2\ns3 ]",
2730
+ nil,
2731
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2732
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2733
+ :tSPACE, nil, EXPR_BEG,
2734
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2735
+ :tSPACE, nil, EXPR_BEG,
2736
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2737
+ :tSPACE, nil, EXPR_BEG,
2738
+ :tSTRING_END, nil, EXPR_LIT)
2515
2739
  end
2516
2740
 
2517
2741
  def test_yylex_string_pct_other
2518
2742
  assert_lex3("%%blah%",
2519
2743
  nil,
2520
- :tSTRING_BEG, "%%", :expr_beg,
2521
- :tSTRING_CONTENT, "blah", :expr_beg,
2522
- :tSTRING_END, "%", :expr_end)
2744
+ :tSTRING_BEG, "%%", EXPR_BEG,
2745
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2746
+ :tSTRING_END, "%", EXPR_LIT)
2747
+ end
2748
+
2749
+ def test_yylex_string_pct_q
2750
+ assert_lex3("%q[s1 s2]",
2751
+ nil,
2752
+ :tSTRING_BEG, "%q[", EXPR_BEG,
2753
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2754
+ :tSTRING_END, "]", EXPR_LIT)
2755
+ end
2756
+
2757
+ def test_yylex_string_pct_s
2758
+ assert_lex3("%s[s1 s2]",
2759
+ nil,
2760
+ :tSYMBEG, "%s[", EXPR_FNAME, # TODO: :tSYM_BEG ?
2761
+ :tSTRING_CONTENT, "s1 s2", EXPR_FNAME, # man... I don't like this
2762
+ :tSTRING_END, "]", EXPR_LIT)
2523
2763
  end
2524
2764
 
2525
2765
  def test_yylex_string_pct_w
@@ -2534,130 +2774,222 @@ class TestRubyLexer < Minitest::Test
2534
2774
  def test_yylex_string_pct_w_bs_nl
2535
2775
  assert_lex3("%w[s1 \\\ns2]",
2536
2776
  nil,
2537
- :tQWORDS_BEG, "%w[", :expr_beg,
2538
- :tSTRING_CONTENT, "s1", :expr_beg,
2539
- :tSPACE, nil, :expr_beg,
2540
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2541
- :tSPACE, nil, :expr_beg,
2542
- :tSTRING_END, nil, :expr_end)
2777
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2778
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2779
+ :tSPACE, nil, EXPR_BEG,
2780
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2781
+ :tSPACE, nil, EXPR_BEG,
2782
+ :tSTRING_END, nil, EXPR_LIT)
2543
2783
  end
2544
2784
 
2545
2785
  def test_yylex_string_pct_w_bs_sp
2546
2786
  assert_lex3("%w[s\\ 1 s\\ 2]",
2547
2787
  nil,
2548
- :tQWORDS_BEG, "%w[", :expr_beg,
2549
- :tSTRING_CONTENT, "s 1", :expr_beg,
2550
- :tSPACE, nil, :expr_beg,
2551
- :tSTRING_CONTENT, "s 2", :expr_beg,
2552
- :tSPACE, nil, :expr_beg,
2553
- :tSTRING_END, nil, :expr_end)
2788
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2789
+ :tSTRING_CONTENT, "s 1", EXPR_BEG,
2790
+ :tSPACE, nil, EXPR_BEG,
2791
+ :tSTRING_CONTENT, "s 2", EXPR_BEG,
2792
+ :tSPACE, nil, EXPR_BEG,
2793
+ :tSTRING_END, nil, EXPR_LIT)
2554
2794
  end
2555
2795
 
2556
2796
  def test_yylex_string_single
2557
- assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
2797
+ assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
2558
2798
  end
2559
2799
 
2560
2800
  def test_yylex_string_single_escape_chars
2561
- assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
2801
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
2562
2802
  end
2563
2803
 
2564
- def test_yylex_string_single_nl
2565
- assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
2804
+ def test_yylex_string_single_escape_quote_and_backslash
2805
+ assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz",
2806
+ EXPR_LIT)
2566
2807
  end
2567
2808
 
2568
2809
  def test_yylex_string_single_escaped_quote
2569
- assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", :expr_end)
2810
+ assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
2570
2811
  end
2571
2812
 
2572
- def test_yylex_symbol
2573
- assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
2813
+ def test_yylex_string_single_nl
2814
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
2574
2815
  end
2575
2816
 
2576
- def test_yylex_symbol_zero_byte
2577
- assert_lex(":\"symbol\0\"", nil,
2578
- :tSYMBOL, "symbol\0", :expr_end)
2817
+ def test_yylex_string_utf8_complex
2818
+ chr = [0x3024].pack("U")
2819
+
2820
+ assert_lex3('"#@a\u{3024}"',
2821
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2822
+ :tSTRING_BEG, '"', EXPR_BEG,
2823
+ :tSTRING_DVAR, nil, EXPR_BEG,
2824
+ :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2825
+ :tSTRING_END, '"', EXPR_LIT)
2826
+ end
2827
+
2828
+ def test_yylex_string_utf8_complex_missing_hex
2829
+ chr = [0x302].pack("U")
2830
+ str = "#{chr}zzz"
2831
+
2832
+ refute_lex('"#@a\u302zzz"',
2833
+ :tSTRING_BEG, '"',
2834
+ :tSTRING_DVAR, nil,
2835
+ :tSTRING_CONTENT, "@a"+str,
2836
+ :tSTRING_END, '"')
2837
+
2838
+ chr = [0x30].pack("U")
2839
+ str = "#{chr}zzz"
2840
+
2841
+ refute_lex('"#@a\u30zzz"',
2842
+ :tSTRING_BEG, '"',
2843
+ :tSTRING_DVAR, nil,
2844
+ :tSTRING_CONTENT, "@a"+str,
2845
+ :tSTRING_END, '"')
2846
+
2847
+ chr = [0x3].pack("U")
2848
+ str = "#{chr}zzz"
2849
+
2850
+ refute_lex('"#@a\u3zzz"',
2851
+ :tSTRING_BEG, '"',
2852
+ :tSTRING_DVAR, nil,
2853
+ :tSTRING_CONTENT, "@a"+str,
2854
+ :tSTRING_END, '"')
2855
+ end
2856
+
2857
+ def test_yylex_string_utf8_bad_encoding_with_escapes
2858
+ str = "\"\\xBADπ\""
2859
+ exp = "\xBADπ".b
2860
+
2861
+ assert_lex(str,
2862
+ s(:str, exp),
2863
+ :tSTRING, exp, EXPR_END)
2864
+ end
2865
+
2866
+ def test_yylex_string_utf8_complex_trailing_hex
2867
+ chr = [0x3024].pack("U")
2868
+ str = "#{chr}abz"
2869
+
2870
+ assert_lex3('"#@a\u3024abz"',
2871
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2872
+ :tSTRING_BEG, '"', EXPR_BEG,
2873
+ :tSTRING_DVAR, nil, EXPR_BEG,
2874
+ :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2875
+ :tSTRING_END, '"', EXPR_LIT)
2876
+ end
2877
+
2878
+ def test_yylex_string_utf8_missing_hex
2879
+ refute_lex('"\u3zzz"')
2880
+ refute_lex('"\u30zzz"')
2881
+ refute_lex('"\u302zzz"')
2882
+ end
2883
+
2884
+ def test_yylex_string_utf8_simple
2885
+ chr = [0x3024].pack("U")
2886
+
2887
+ assert_lex3('"\u{3024}"',
2888
+ s(:str, chr),
2889
+ :tSTRING, chr, EXPR_END)
2890
+ end
2891
+
2892
+ def test_yylex_string_utf8_trailing_hex
2893
+ chr = [0x3024].pack("U")
2894
+ str = "#{chr}abz"
2895
+
2896
+ assert_lex3('"\u3024abz"',
2897
+ s(:str, str),
2898
+ :tSTRING, str, EXPR_END)
2899
+ end
2900
+
2901
+ def test_yylex_sym_quoted
2902
+ assert_lex(":'a'",
2903
+ s(:lit, :a),
2904
+
2905
+ :tSYMBOL, "a", EXPR_LIT, 0, 0)
2906
+ end
2907
+
2908
+ def test_yylex_symbol
2909
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_LIT)
2579
2910
  end
2580
2911
 
2581
2912
  def test_yylex_symbol_double
2582
2913
  assert_lex3(":\"symbol\"",
2583
2914
  nil,
2584
- :tSYMBOL, "symbol", :expr_end)
2915
+ :tSYMBOL, "symbol", EXPR_LIT)
2585
2916
  end
2586
2917
 
2587
2918
  def test_yylex_symbol_double_interp
2588
2919
  assert_lex3(':"symbol#{1+1}"',
2589
2920
  nil,
2590
- :tSYMBEG, ":", :expr_fname,
2591
- :tSTRING_CONTENT, "symbol", :expr_fname,
2592
- :tSTRING_DBEG, nil, :expr_fname,
2593
- :tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
2594
- :tSTRING_END, "\"", :expr_end)
2921
+ :tSYMBEG, ":", EXPR_FNAME,
2922
+ :tSTRING_CONTENT, "symbol", EXPR_FNAME,
2923
+ :tSTRING_DBEG, nil, EXPR_FNAME,
2924
+ :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
2925
+ :tSTRING_END, "\"", EXPR_LIT)
2595
2926
  end
2596
2927
 
2597
2928
  def test_yylex_symbol_single
2598
2929
  assert_lex3(":'symbol'",
2599
2930
  nil,
2600
- :tSYMBOL, "symbol", :expr_end)
2931
+ :tSYMBOL, "symbol", EXPR_LIT)
2601
2932
  end
2602
2933
 
2603
- def test_yylex_symbol_single_noninterp
2604
- assert_lex3(':\'symbol#{1+1}\'',
2934
+ def test_yylex_symbol_single_escape_chars
2935
+ assert_lex3(":'s\\tri\\ng'",
2605
2936
  nil,
2606
- :tSYMBOL, 'symbol#{1+1}', :expr_end)
2937
+ :tSYMBOL, "s\\tri\\ng", EXPR_LIT)
2607
2938
  end
2608
2939
 
2609
- def test_yylex_symbol_single_escape_chars
2610
- assert_lex3(":'s\\tri\\ng'",
2940
+ def test_yylex_symbol_single_noninterp
2941
+ assert_lex3(':\'symbol#{1+1}\'',
2611
2942
  nil,
2612
- :tSYMBOL, "s\\tri\\ng", :expr_end)
2943
+ :tSYMBOL, 'symbol#{1+1}', EXPR_LIT)
2613
2944
  end
2614
2945
 
2615
- def test_yylex_string_single_escape_quote_and_backslash
2616
- assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz", :expr_end)
2946
+ def test_yylex_symbol_zero_byte
2947
+ assert_lex(":\"symbol\0\"", nil,
2948
+ :tSYMBOL, "symbol\0", EXPR_LIT)
2617
2949
  end
2618
2950
 
2619
2951
  def test_yylex_ternary1
2620
2952
  assert_lex3("a ? b : c",
2621
2953
  nil,
2622
- :tIDENTIFIER, "a", :expr_cmdarg,
2623
- :tEH, "?", :expr_value,
2624
- :tIDENTIFIER, "b", :expr_arg,
2625
- :tCOLON, ":", :expr_beg,
2626
- :tIDENTIFIER, "c", :expr_arg)
2954
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2955
+ :tEH, "?", EXPR_BEG,
2956
+ :tIDENTIFIER, "b", EXPR_ARG,
2957
+ :tCOLON, ":", EXPR_BEG,
2958
+ :tIDENTIFIER, "c", EXPR_ARG)
2627
2959
 
2628
2960
  assert_lex3("a ?bb : c", # GAH! MATZ!!!
2629
2961
  nil,
2630
- :tIDENTIFIER, "a", :expr_cmdarg,
2631
- :tEH, "?", :expr_beg,
2632
- :tIDENTIFIER, "bb", :expr_arg,
2633
- :tCOLON, ":", :expr_beg,
2634
- :tIDENTIFIER, "c", :expr_arg)
2962
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2963
+ :tEH, "?", EXPR_BEG,
2964
+ :tIDENTIFIER, "bb", EXPR_ARG,
2965
+ :tCOLON, ":", EXPR_BEG,
2966
+ :tIDENTIFIER, "c", EXPR_ARG)
2635
2967
 
2636
2968
  assert_lex3("42 ?",
2637
2969
  nil,
2638
- :tINTEGER, 42, :expr_end,
2639
- :tEH, "?", :expr_value)
2970
+ :tINTEGER, 42, EXPR_NUM,
2971
+ :tEH, "?", EXPR_BEG)
2640
2972
  end
2641
2973
 
2642
2974
  def test_yylex_tilde
2643
- assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
2975
+ assert_lex3("~", nil, :tTILDE, "~", EXPR_BEG)
2644
2976
  end
2645
2977
 
2646
2978
  def test_yylex_tilde_unary
2647
- self.lex_state = :expr_fname
2979
+ self.lex_state = EXPR_FNAME
2648
2980
 
2649
- assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
2981
+ assert_lex3("~@", nil, :tTILDE, "~", EXPR_ARG)
2650
2982
  end
2651
2983
 
2652
2984
  def test_yylex_uminus
2653
2985
  assert_lex3("-blah",
2654
2986
  nil,
2655
- :tUMINUS, "-", :expr_beg,
2656
- :tIDENTIFIER, "blah", :expr_arg)
2987
+ :tUMINUS, "-", EXPR_BEG,
2988
+ :tIDENTIFIER, "blah", EXPR_ARG)
2657
2989
  end
2658
2990
 
2659
2991
  def test_yylex_underscore
2660
- assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
2992
+ assert_lex3("_var", nil, :tIDENTIFIER, "_var", EXPR_CMDARG)
2661
2993
  end
2662
2994
 
2663
2995
  def test_yylex_underscore_end
@@ -2669,229 +3001,45 @@ class TestRubyLexer < Minitest::Test
2669
3001
  def test_yylex_uplus
2670
3002
  assert_lex3("+blah",
2671
3003
  nil,
2672
- :tUPLUS, "+", :expr_beg,
2673
- :tIDENTIFIER, "blah", :expr_arg)
3004
+ :tUPLUS, "+", EXPR_BEG,
3005
+ :tIDENTIFIER, "blah", EXPR_ARG)
2674
3006
  end
2675
3007
 
2676
3008
  def test_zbug_float_in_decl
2677
3009
  assert_lex3("def initialize(u = 0.0, s = 0.0",
2678
3010
  nil,
2679
- :kDEF, "def", :expr_fname,
2680
- :tIDENTIFIER, "initialize", :expr_endfn,
2681
- :tLPAREN2, "(", :expr_beg,
2682
- :tIDENTIFIER, "u", :expr_arg,
2683
- :tEQL, "=", :expr_beg,
2684
- :tFLOAT, 0.0, :expr_end,
2685
- :tCOMMA, ",", :expr_beg,
2686
- :tIDENTIFIER, "s", :expr_arg,
2687
- :tEQL, "=", :expr_beg,
2688
- :tFLOAT, 0.0, :expr_end)
3011
+ :kDEF, "def", EXPR_FNAME,
3012
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3013
+ :tLPAREN2, "(", EXPR_PAR,
3014
+ :tIDENTIFIER, "u", EXPR_ARG,
3015
+ :tEQL, "=", EXPR_BEG,
3016
+ :tFLOAT, 0.0, EXPR_NUM,
3017
+ :tCOMMA, ",", EXPR_PAR,
3018
+ :tIDENTIFIER, "s", EXPR_ARG,
3019
+ :tEQL, "=", EXPR_BEG,
3020
+ :tFLOAT, 0.0, EXPR_NUM)
2689
3021
  end
2690
3022
 
2691
3023
  def test_zbug_id_equals
2692
3024
  assert_lex3("a = 0.0",
2693
3025
  nil,
2694
- :tIDENTIFIER, "a", :expr_cmdarg,
2695
- :tEQL, "=", :expr_beg,
2696
- :tFLOAT, 0.0, :expr_end)
3026
+ :tIDENTIFIER, "a", EXPR_CMDARG,
3027
+ :tEQL, "=", EXPR_BEG,
3028
+ :tFLOAT, 0.0, EXPR_NUM)
2697
3029
  end
2698
3030
 
2699
3031
  def test_zbug_no_spaces_in_decl
2700
3032
  assert_lex3("def initialize(u=0.0,s=0.0",
2701
3033
  nil,
2702
- :kDEF, "def", :expr_fname,
2703
- :tIDENTIFIER, "initialize", :expr_endfn,
2704
- :tLPAREN2, "(", :expr_beg,
2705
- :tIDENTIFIER, "u", :expr_arg,
2706
- :tEQL, "=", :expr_beg,
2707
- :tFLOAT, 0.0, :expr_end,
2708
- :tCOMMA, ",", :expr_beg,
2709
- :tIDENTIFIER, "s", :expr_arg,
2710
- :tEQL, "=", :expr_beg,
2711
- :tFLOAT, 0.0, :expr_end)
2712
- end
2713
-
2714
- def test_pct_w_backslashes
2715
- ["\t", "\n", "\r", "\v", "\f"].each do |char|
2716
- next if !RubyLexer::HAS_ENC and char == "\v"
2717
-
2718
- assert_lex("%w[foo#{char}bar]",
2719
- s(:array, s(:str, "foo"), s(:str, "bar")),
2720
-
2721
- :tQWORDS_BEG, "%w[", :expr_beg, 0, 0,
2722
- :tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
2723
- :tSPACE, nil, :expr_beg, 0, 0,
2724
- :tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
2725
- :tSPACE, nil, :expr_beg, 0, 0,
2726
- :tSTRING_END, nil, :expr_end, 0, 0)
2727
- end
2728
- end
2729
-
2730
- def test_yylex_sym_quoted
2731
- assert_lex(":'a'",
2732
- s(:lit, :a),
2733
-
2734
- :tSYMBOL, "a", :expr_end, 0, 0)
2735
- end
2736
-
2737
- def test_yylex_hash_colon
2738
- assert_lex("{a:1}",
2739
- s(:hash, s(:lit, :a), s(:lit, 1)),
2740
-
2741
- :tLBRACE, "{", :expr_beg, 0, 1,
2742
- :tLABEL, "a", :expr_labeled, 0, 1,
2743
- :tINTEGER, 1, :expr_end, 0, 1,
2744
- :tRCURLY, "}", :expr_endarg, 0, 0)
2745
- end
2746
-
2747
- def test_yylex_hash_colon_quoted_22
2748
- setup_lexer_class RubyParser::V22
2749
-
2750
- assert_lex("{'a':1}",
2751
- s(:hash, s(:lit, :a), s(:lit, 1)),
2752
-
2753
- :tLBRACE, "{", :expr_beg, 0, 1,
2754
- :tLABEL, "a", :expr_labeled, 0, 1,
2755
- :tINTEGER, 1, :expr_end, 0, 1,
2756
- :tRCURLY, "}", :expr_endarg, 0, 0)
2757
- end
2758
-
2759
- def test_yylex_hash_colon_quoted_symbol_22
2760
- setup_lexer_class RubyParser::V22
2761
-
2762
- assert_lex("{'abc': :b}",
2763
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2764
-
2765
- :tLBRACE, "{", :expr_beg, 0, 1,
2766
- :tLABEL, "abc", :expr_labeled, 0, 1,
2767
- :tSYMBOL, "b", :expr_end, 0, 1,
2768
- :tRCURLY, "}", :expr_endarg, 0, 0)
2769
- end
2770
-
2771
- def test_yylex_hash_colon_double_quoted_symbol_22
2772
- setup_lexer_class RubyParser::V22
2773
-
2774
- assert_lex('{"abc": :b}',
2775
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2776
-
2777
- :tLBRACE, "{", :expr_beg, 0, 1,
2778
- :tLABEL, "abc", :expr_labeled, 0, 1,
2779
- :tSYMBOL, "b", :expr_end, 0, 1,
2780
- :tRCURLY, "}", :expr_endarg, 0, 0)
2781
- end
2782
-
2783
- def test_yylex_required_kwarg_no_value_22
2784
- setup_lexer_class RubyParser::V22
2785
-
2786
- assert_lex3("def foo a:, b:\nend",
2787
- nil,
2788
- :kDEF, "def", :expr_fname,
2789
- :tIDENTIFIER, "foo", :expr_endfn,
2790
- :tLABEL, "a", :expr_labeled,
2791
- :tCOMMA, ",", :expr_beg,
2792
- :tLABEL, "b", :expr_labeled,
2793
- :kEND, "end", :expr_end)
2794
- end
2795
-
2796
- def test_yylex_hash_colon_double_quoted_with_escapes
2797
- setup_lexer_class RubyParser::V22
2798
-
2799
- assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
2800
- nil,
2801
-
2802
- :tLBRACE, "{", :expr_beg,
2803
- :tLABEL, "s\tr\i\ng\\foo'bar", :expr_labeled,
2804
- :tINTEGER, 1, :expr_end,
2805
- :tRCURLY, "}", :expr_endarg)
2806
- end
2807
-
2808
- def test_yylex_hash_colon_quoted_with_escapes
2809
- setup_lexer_class RubyParser::V22
2810
-
2811
- assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
2812
- nil,
2813
-
2814
- :tLBRACE, "{", :expr_beg,
2815
- :tLABEL, "s\\tr\\i\\ng\\foo'bar", :expr_labeled,
2816
- :tINTEGER, 1, :expr_end,
2817
- :tRCURLY, "}", :expr_endarg)
2818
- end
2819
-
2820
- def test_ruby21_rational_literal
2821
- setup_lexer_class RubyParser::V21
2822
-
2823
- assert_lex3("10r", nil, :tRATIONAL, Rational(10), :expr_end)
2824
- assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), :expr_end)
2825
- assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), :expr_end)
2826
- assert_lex3("0or", nil, :tRATIONAL, Rational(0), :expr_end)
2827
- assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), :expr_end)
2828
- assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), :expr_end)
2829
- assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), :expr_end)
2830
- assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), :expr_end)
2831
- assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), :expr_end)
2832
- assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), :expr_end)
2833
-
2834
- assert_lex3("-10r", nil,
2835
- :tUMINUS_NUM, "-", :expr_beg,
2836
- :tRATIONAL, Rational(10), :expr_end)
2837
- end
2838
-
2839
- def test_ruby21_imaginary_literal
2840
- setup_lexer_class RubyParser::V21
2841
-
2842
- assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), :expr_end)
2843
- assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), :expr_end)
2844
- assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), :expr_end)
2845
- assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), :expr_end)
2846
- assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), :expr_end)
2847
- assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
2848
- assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
2849
- assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
2850
- assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
2851
- assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
2852
-
2853
- assert_lex3("-10i", nil,
2854
- :tUMINUS_NUM, "-", :expr_beg,
2855
- :tIMAGINARY, Complex(0, 10), :expr_end)
2856
- end
2857
-
2858
- def test_ruby21_rational_imaginary_literal
2859
- setup_lexer_class RubyParser::V21
2860
-
2861
- assert_lex3("1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
2862
- assert_lex3("0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
2863
- assert_lex3("0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
2864
- assert_lex3("0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
2865
- assert_lex3("0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
2866
- assert_lex3("1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
2867
- assert_lex3("15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
2868
- assert_lex3("15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
2869
- assert_lex3("1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
2870
- assert_lex3("1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
2871
-
2872
- assert_lex3("-10ri", nil,
2873
- :tUMINUS_NUM, "-", :expr_beg,
2874
- :tIMAGINARY, Complex(0, Rational(10)), :expr_end)
2875
- end
2876
-
2877
- def test_ruby21_imaginary_literal_with_succeeding_keyword
2878
- skip "Currently does not tokenize correctly"
2879
-
2880
- setup_lexer_class RubyParser::V21
2881
-
2882
- assert_lex3("1if", nil,
2883
- :tINTEGER, 1, :expr_end,
2884
- :kIF_MOD, "if", :expr_beg)
2885
- assert_lex3("1rif", nil,
2886
- :tRATIONAL, Rational(1), :expr_end,
2887
- :kIF_MOD, "if", :expr_beg)
2888
- assert_lex3("1.0if", nil,
2889
- :tFLOAT, 1.0, :expr_end,
2890
- :kIF_MOD, "if", :expr_beg)
2891
- assert_lex3("1.0rif", nil,
2892
- :tRATIONAL, Rational("1.0"), :expr_end,
2893
- :kIF_MOD, "if", :expr_beg)
2894
-
2895
- flunk
3034
+ :kDEF, "def", EXPR_FNAME,
3035
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3036
+ :tLPAREN2, "(", EXPR_PAR,
3037
+ :tIDENTIFIER, "u", EXPR_ARG,
3038
+ :tEQL, "=", EXPR_BEG,
3039
+ :tFLOAT, 0.0, EXPR_NUM,
3040
+ :tCOMMA, ",", EXPR_PAR,
3041
+ :tIDENTIFIER, "s", EXPR_ARG,
3042
+ :tEQL, "=", EXPR_BEG,
3043
+ :tFLOAT, 0.0, EXPR_NUM)
2896
3044
  end
2897
3045
  end