ruby_parser 3.13.0 → 3.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.autotest +18 -29
  5. data/History.rdoc +123 -0
  6. data/Manifest.txt +2 -0
  7. data/README.rdoc +3 -3
  8. data/Rakefile +41 -24
  9. data/bin/ruby_parse +3 -1
  10. data/bin/ruby_parse_extract_error +18 -35
  11. data/compare/normalize.rb +43 -3
  12. data/debugging.md +39 -0
  13. data/lib/rp_extensions.rb +1 -1
  14. data/lib/ruby20_parser.rb +3654 -3466
  15. data/lib/ruby20_parser.y +504 -327
  16. data/lib/ruby21_parser.rb +3643 -3455
  17. data/lib/ruby21_parser.y +512 -334
  18. data/lib/ruby22_parser.rb +3669 -3492
  19. data/lib/ruby22_parser.y +513 -335
  20. data/lib/ruby23_parser.rb +3692 -3499
  21. data/lib/ruby23_parser.y +513 -335
  22. data/lib/ruby24_parser.rb +3685 -3463
  23. data/lib/ruby24_parser.y +517 -331
  24. data/lib/ruby25_parser.rb +3685 -3462
  25. data/lib/ruby25_parser.y +517 -331
  26. data/lib/ruby26_parser.rb +3696 -3471
  27. data/lib/ruby26_parser.y +523 -335
  28. data/lib/ruby27_parser.rb +7224 -0
  29. data/lib/ruby27_parser.y +2657 -0
  30. data/lib/ruby_lexer.rb +611 -495
  31. data/lib/ruby_lexer.rex +27 -28
  32. data/lib/ruby_lexer.rex.rb +71 -31
  33. data/lib/ruby_parser.rb +31 -27
  34. data/lib/ruby_parser.yy +529 -336
  35. data/lib/ruby_parser_extras.rb +720 -449
  36. data/test/test_ruby_lexer.rb +1560 -1412
  37. data/test/test_ruby_parser.rb +2611 -1912
  38. data/test/test_ruby_parser_extras.rb +39 -4
  39. data/tools/munge.rb +12 -6
  40. data/tools/ripper.rb +19 -3
  41. metadata +25 -18
  42. metadata.gz.sig +4 -1
data/test/test_ruby_lexer.rb CHANGED
@@ -1,29 +1,24 @@
1
- # encoding: US-ASCII
2
-
3
1
  require "minitest/autorun"
4
2
  require "ruby_lexer"
5
3
  require "ruby_parser"
6
4
 
7
5
  class TestRubyLexer < Minitest::Test
6
+ include RubyLexer::State::Values
7
+
8
8
  attr_accessor :processor, :lex, :parser_class, :lex_state
9
9
 
10
- alias :lexer :lex # lets me copy/paste code from parser
11
- alias :lexer= :lex=
10
+ alias lexer lex # lets me copy/paste code from parser
11
+ alias lexer= lex=
12
12
 
13
13
  def setup
14
- self.lex_state = :expr_beg
14
+ self.lex_state = EXPR_BEG
15
15
  setup_lexer_class RubyParser.latest.class
16
16
  end
17
17
 
18
18
  def setup_lexer input, exp_sexp = nil
19
19
  setup_new_parser
20
20
  lex.ss = RPStringScanner.new(input)
21
- lex.lex_state = self.lex_state
22
- end
23
-
24
- def setup_new_parser
25
- self.processor = parser_class.new
26
- self.lex = processor.lexer
21
+ lex.lex_state = lex_state
27
22
  end
28
23
 
29
24
  def setup_lexer_class parser_class
@@ -32,11 +27,16 @@ class TestRubyLexer < Minitest::Test
32
27
  setup_lexer "blah blah"
33
28
  end
34
29
 
35
- def assert_lex input, exp_sexp, *args, &b
30
+ def setup_new_parser
31
+ self.processor = parser_class.new
32
+ self.lex = processor.lexer
33
+ end
34
+
35
+ def assert_lex input, exp_sexp, *args
36
36
  setup_lexer input
37
37
  assert_parse input, exp_sexp if exp_sexp
38
38
 
39
- b.call if b
39
+ yield if block_given?
40
40
 
41
41
  args.each_slice(5) do |token, value, state, paren, brace|
42
42
  assert_next_lexeme token, value, state, paren, brace
@@ -51,48 +51,41 @@ class TestRubyLexer < Minitest::Test
51
51
  assert_lex(input, exp_sexp, *args, &block)
52
52
  end
53
53
 
54
- def refute_lex input, *args # TODO: re-sort
55
- args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
56
-
57
- assert_raises RubyParser::SyntaxError do
58
- assert_lex(input, nil, *args)
59
- end
60
- end
61
-
62
- def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
54
+ def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
63
55
  assert_lex3("def #{name} ",
64
56
  nil,
65
57
 
66
- :kDEF, "def", :expr_fname,
58
+ :kDEF, "def", EXPR_FNAME,
67
59
  type, name, end_state)
68
60
  end
69
61
 
70
62
  def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
71
63
  adv = @lex.next_token
72
64
 
73
- assert adv, "no more tokens"
65
+ assert adv, "no more tokens, expecting: %p %p %p %p %p" % [token, value, state, paren, brace]
74
66
 
75
67
  act_token, act_value = adv
76
68
 
77
69
  msg = message {
78
- act = [act_token, act_value, @lex.lex_state,
79
- @lex.paren_nest, @lex.brace_nest]
70
+ act = [act_token, act_value, @lex.lex_state, @lex.paren_nest, @lex.brace_nest]
80
71
  exp = [token, value, state, paren, brace]
81
72
  "#{exp.inspect} vs #{act.inspect}"
82
73
  }
83
74
 
84
75
  act_value = act_value.first if Array === act_value
85
76
 
86
- assert_equal token, act_token, msg
77
+ assert_equal token, act_token, msg
87
78
  case value
88
79
  when Float then
89
80
  assert_in_epsilon value, act_value, 0.001, msg
90
81
  when NilClass then
91
82
  assert_nil act_value, msg
83
+ when String then
84
+ assert_equal value, act_value.b.force_encoding(value.encoding), msg
92
85
  else
93
- assert_equal value, act_value, msg
86
+ assert_equal value, act_value, msg
94
87
  end
95
- assert_equal state, @lex.lex_state, msg if state
88
+ assert_match state, @lex.lex_state, msg if state
96
89
  assert_equal paren, @lex.paren_nest, msg if paren
97
90
  assert_equal brace, @lex.brace_nest, msg if brace
98
91
  end
@@ -102,8 +95,9 @@ class TestRubyLexer < Minitest::Test
102
95
  end
103
96
 
104
97
  def assert_read_escape expected, input
105
- @lex.ss.string = input
106
- assert_equal expected, @lex.read_escape, input
98
+ @lex.ss.string = input.dup
99
+ enc = expected.encoding
100
+ assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
107
101
  end
108
102
 
109
103
  def assert_read_escape_bad input # TODO: rename refute_read_escape
@@ -113,6 +107,20 @@ class TestRubyLexer < Minitest::Test
113
107
  end
114
108
  end
115
109
 
110
+ def refute_lex input, *args # TODO: re-sort
111
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
112
+
113
+ assert_raises RubyParser::SyntaxError do
114
+ assert_lex(input, nil, *args)
115
+ end
116
+ end
117
+
118
+ def refute_lex5 input, *args
119
+ assert_raises RubyParser::SyntaxError do
120
+ assert_lex(input, *args)
121
+ end
122
+ end
123
+
116
124
  def refute_lexeme
117
125
  x = y = @lex.next_token
118
126
 
@@ -132,15 +140,15 @@ class TestRubyLexer < Minitest::Test
132
140
  lexer.cmdarg.push false
133
141
 
134
142
  lexer.lex_strterm = nil
135
- lexer.lex_state = :expr_beg
143
+ lexer.lex_state = EXPR_BEG
136
144
 
137
145
  yield
138
146
 
139
- lexer.lex_state = :expr_endarg
140
- assert_next_lexeme :tSTRING_DEND, "}", :expr_endarg, 0
147
+ lexer.lex_state = EXPR_ENDARG
148
+ assert_next_lexeme :tSTRING_DEND, "}", EXPR_END|EXPR_ENDARG, 0
141
149
 
142
150
  lexer.lex_strterm = lex_strterm
143
- lexer.lex_state = :expr_beg
151
+ lexer.lex_state = EXPR_BEG
144
152
  lexer.string_nest = string_nest
145
153
  lexer.brace_nest = brace_nest
146
154
 
@@ -156,9 +164,20 @@ class TestRubyLexer < Minitest::Test
156
164
  assert_nil @lex.next_token
157
165
  end
158
166
 
159
- def test_unicode_ident
160
- s = "@\u1088\u1077\u1093\u1072"
161
- assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
167
+ def test_pct_w_backslashes
168
+ ["\t", "\n", "\r", "\v", "\f"].each do |char|
169
+ next if !RubyLexer::HAS_ENC and char == "\v"
170
+
171
+ assert_lex("%w[foo#{char}bar]",
172
+ s(:array, s(:str, "foo"), s(:str, "bar")),
173
+
174
+ :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
175
+ :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
176
+ :tSPACE, nil, EXPR_BEG, 0, 0,
177
+ :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
178
+ :tSPACE, nil, EXPR_BEG, 0, 0,
179
+ :tSTRING_END, nil, EXPR_LIT, 0, 0)
180
+ end
162
181
  end
163
182
 
164
183
  def test_read_escape
@@ -214,12 +233,96 @@ class TestRubyLexer < Minitest::Test
214
233
  assert_read_escape "\230", 'M-\cx'
215
234
  end
216
235
 
236
+ def test_ruby21_imaginary_literal
237
+ setup_lexer_class RubyParser::V21
238
+
239
+ assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
240
+ assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
241
+ assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
242
+ assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
243
+ assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
244
+ assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
245
+ assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
246
+ assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
247
+ assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
248
+ assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
249
+
250
+ c010 = Complex(0, 10)
251
+ assert_lex3("-10i", nil,
252
+ :tUMINUS_NUM, "-", EXPR_BEG,
253
+ :tIMAGINARY, c010, EXPR_NUM)
254
+ end
255
+
256
+ def test_ruby21_imaginary_literal_with_succeeding_keyword
257
+ setup_lexer_class RubyParser::V21
258
+
259
+ # 2/4 scenarios are syntax errors on all tested versions so I
260
+ # deleted them.
261
+
262
+ assert_lex3("1if", nil,
263
+ :tINTEGER, 1, EXPR_NUM,
264
+ :kIF_MOD, "if", EXPR_PAR)
265
+ assert_lex3("1.0if", nil,
266
+ :tFLOAT, 1.0, EXPR_NUM,
267
+ :kIF_MOD, "if", EXPR_PAR)
268
+ end
269
+
270
+ def test_ruby21_rational_imaginary_literal
271
+ setup_lexer_class RubyParser::V21
272
+
273
+ assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
274
+ assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
275
+ assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
276
+ assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
277
+ assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
278
+ assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
279
+ assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
280
+ assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
281
+ assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
282
+ assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
283
+
284
+ assert_lex3("-10ri", nil,
285
+ :tUMINUS_NUM, "-", EXPR_BEG,
286
+ :tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
287
+ end
288
+
289
+ def test_ruby21_rational_literal
290
+ setup_lexer_class RubyParser::V21
291
+
292
+ assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
293
+ assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
294
+ assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
295
+ assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
296
+ assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
297
+ assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
298
+ assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
299
+ assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
300
+ assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
301
+ assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
302
+
303
+ r10 = Rational(10)
304
+ assert_lex3("-10r", nil,
305
+ :tUMINUS_NUM, "-", EXPR_BEG,
306
+ :tRATIONAL, r10, EXPR_NUM)
307
+ end
308
+
309
+ def test_unicode_ident
310
+ s = "@\u1088\u1077\u1093\u1072"
311
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
312
+ end
313
+
314
+ def test_why_does_ruby_hate_me?
315
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
316
+ nil,
317
+ :tSTRING, %W[ Nl% \u0000 \u0000 A \u0000 999 ].join, EXPR_END)
318
+ end
319
+
217
320
  def test_yylex_ambiguous_uminus
218
321
  assert_lex3("m -3",
219
322
  nil,
220
- :tIDENTIFIER, "m", :expr_cmdarg,
221
- :tUMINUS_NUM, "-", :expr_beg,
222
- :tINTEGER, 3, :expr_end)
323
+ :tIDENTIFIER, "m", EXPR_CMDARG,
324
+ :tUMINUS_NUM, "-", EXPR_BEG,
325
+ :tINTEGER, 3, EXPR_NUM)
223
326
 
224
327
  # TODO: verify warning
225
328
  end
@@ -227,36 +330,45 @@ class TestRubyLexer < Minitest::Test
227
330
  def test_yylex_ambiguous_uplus
228
331
  assert_lex3("m +3",
229
332
  nil,
230
- :tIDENTIFIER, "m", :expr_cmdarg,
231
- :tINTEGER, 3, :expr_end)
333
+ :tIDENTIFIER, "m", EXPR_CMDARG,
334
+ :tINTEGER, 3, EXPR_NUM)
232
335
 
233
336
  # TODO: verify warning
234
337
  end
235
338
 
236
339
  def test_yylex_and
237
- assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
340
+ assert_lex3("&", nil, :tAMPER, "&", EXPR_BEG)
238
341
  end
239
342
 
240
343
  def test_yylex_and2
241
- assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
344
+ assert_lex3("&&", nil, :tANDOP, "&&", EXPR_BEG)
242
345
  end
243
346
 
244
347
  def test_yylex_and2_equals
245
- assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
348
+ assert_lex3("&&=", nil, :tOP_ASGN, "&&", EXPR_BEG)
349
+ end
350
+
351
+ def test_yylex_and_arg
352
+ self.lex_state = EXPR_ARG
353
+
354
+ assert_lex3(" &y",
355
+ nil,
356
+ :tAMPER, "&", EXPR_BEG,
357
+ :tIDENTIFIER, "y", EXPR_ARG)
246
358
  end
247
359
 
248
360
  def test_yylex_and_dot
249
361
  setup_lexer_class RubyParser::V23
250
362
 
251
- assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
363
+ assert_lex3("&.", nil, :tLONELY, "&.", EXPR_DOT)
252
364
  end
253
365
 
254
366
  def test_yylex_and_dot_call
255
367
  setup_lexer_class RubyParser::V23
256
368
 
257
369
  assert_lex3("x&.y", nil,
258
- :tIDENTIFIER, "x", :expr_cmdarg,
259
- :tLONELY, "&.", :expr_dot,
370
+ :tIDENTIFIER, "x", EXPR_CMDARG,
371
+ :tLONELY, "&.", EXPR_DOT,
260
372
  :tIDENTIFIER, "y")
261
373
  end
262
374
 
@@ -264,32 +376,23 @@ class TestRubyLexer < Minitest::Test
264
376
  setup_lexer_class Ruby23Parser
265
377
 
266
378
  assert_lex3("x\n&.y", nil,
267
- :tIDENTIFIER, "x", :expr_cmdarg,
268
- :tLONELY, "&.", :expr_dot,
379
+ :tIDENTIFIER, "x", EXPR_CMDARG,
380
+ :tLONELY, "&.", EXPR_DOT,
269
381
  :tIDENTIFIER, "y")
270
382
  end
271
383
 
272
- def test_yylex_and_arg
273
- self.lex_state = :expr_arg
274
-
275
- assert_lex3(" &y",
276
- nil,
277
- :tAMPER, "&", :expr_beg,
278
- :tIDENTIFIER, "y", :expr_arg)
279
- end
280
-
281
384
  def test_yylex_and_equals
282
- assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
385
+ assert_lex3("&=", nil, :tOP_ASGN, "&", EXPR_BEG)
283
386
  end
284
387
 
285
388
  def test_yylex_and_expr
286
- self.lex_state = :expr_arg
389
+ self.lex_state = EXPR_ARG
287
390
 
288
391
  assert_lex3("x & y",
289
392
  nil,
290
- :tIDENTIFIER, "x", :expr_cmdarg,
291
- :tAMPER2, "&", :expr_beg,
292
- :tIDENTIFIER, "y", :expr_arg)
393
+ :tIDENTIFIER, "x", EXPR_CMDARG,
394
+ :tAMPER2, "&", EXPR_BEG,
395
+ :tIDENTIFIER, "y", EXPR_ARG)
293
396
  end
294
397
 
295
398
  def test_yylex_and_meth
@@ -297,338 +400,26 @@ class TestRubyLexer < Minitest::Test
297
400
  end
298
401
 
299
402
  def test_yylex_assoc
300
- assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
301
- end
302
-
303
- def test_yylex_label__20
304
- setup_lexer_class RubyParser::V20
305
-
306
- assert_lex3("{a:",
307
- nil,
308
- :tLBRACE, "{", :expr_beg,
309
- :tLABEL, "a", :expr_labeled)
310
- end
311
-
312
- def test_yylex_label_in_params__20
313
- setup_lexer_class RubyParser::V20
314
-
315
- assert_lex3("foo(a:",
316
- nil,
317
- :tIDENTIFIER, "foo", :expr_cmdarg,
318
- :tLPAREN2, "(", :expr_beg,
319
- :tLABEL, "a", :expr_labeled)
320
- end
321
-
322
- def test_yylex_paren_string_parens_interpolated
323
- setup_lexer('%((#{b}#{d}))',
324
- s(:dstr,
325
- "(",
326
- s(:evstr, s(:call, nil, :b)),
327
- s(:evstr, s(:call, nil, :d)),
328
- s(:str, ")")))
329
-
330
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
331
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
332
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
333
-
334
- emulate_string_interpolation do
335
- assert_next_lexeme :tIDENTIFIER, "b", :expr_cmdarg, 0, 0
336
- end
337
-
338
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
339
-
340
- emulate_string_interpolation do
341
- assert_next_lexeme :tIDENTIFIER, "d", :expr_cmdarg, 0, 0
342
- end
343
-
344
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
345
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
346
-
347
- refute_lexeme
348
- end
349
-
350
- def test_yylex_paren_string_interpolated_regexp
351
- setup_lexer('%( #{(/abcd/)} )',
352
- s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
353
-
354
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
355
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
356
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
357
-
358
- emulate_string_interpolation do
359
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
360
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
361
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
362
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
363
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
364
- end
365
-
366
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
367
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
368
-
369
- refute_lexeme
370
- end
371
-
372
- def test_yylex_not_at_defn__20
373
- setup_lexer_class RubyParser::V20
374
-
375
- assert_lex("def +@; end",
376
- s(:defn, :+@, s(:args), s(:nil)),
377
-
378
- :kDEF, "def", :expr_fname, 0, 0,
379
- :tUPLUS, "+@", :expr_arg, 0, 0,
380
- :tSEMI, ";", :expr_beg, 0, 0,
381
- :kEND, "end", :expr_end, 0, 0)
382
-
383
- assert_lex("def !@; end",
384
- s(:defn, :"!@", s(:args), s(:nil)),
385
-
386
- :kDEF, "def", :expr_fname, 0, 0,
387
- :tUBANG, "!@", :expr_arg, 0, 0,
388
- :tSEMI, ";", :expr_beg, 0, 0,
389
- :kEND, "end", :expr_end, 0, 0)
390
- end
391
-
392
- def test_yylex_not_at_ivar
393
- assert_lex("!@ivar",
394
- s(:call, s(:ivar, :@ivar), :"!"),
395
-
396
- :tBANG, "!", :expr_beg, 0, 0,
397
- :tIVAR, "@ivar", :expr_end, 0, 0)
398
- end
399
-
400
- def test_yylex_number_times_ident_times_return_number
401
- assert_lex("1 * b * 3",
402
- s(:call,
403
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
404
- :*, s(:lit, 3)),
405
-
406
- :tINTEGER, 1, :expr_end, 0, 0,
407
- :tSTAR2, "*", :expr_beg, 0, 0,
408
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
409
- :tSTAR2, "*", :expr_beg, 0, 0,
410
- :tINTEGER, 3, :expr_end, 0, 0)
411
-
412
- assert_lex("1 * b *\n 3",
413
- s(:call,
414
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
415
- :*, s(:lit, 3)),
416
-
417
- :tINTEGER, 1, :expr_end, 0, 0,
418
- :tSTAR2, "*", :expr_beg, 0, 0,
419
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
420
- :tSTAR2, "*", :expr_beg, 0, 0,
421
- :tINTEGER, 3, :expr_end, 0, 0)
422
- end
423
-
424
- def test_yylex_paren_string_parens_interpolated_regexp
425
- setup_lexer('%((#{(/abcd/)}))',
426
- s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
427
-
428
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
429
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
430
-
431
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
432
-
433
- emulate_string_interpolation do
434
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
435
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
436
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
437
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
438
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
439
- end
440
-
441
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
442
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
443
-
444
- refute_lexeme
445
- end
446
-
447
- def test_yylex_method_parens_chevron
448
- assert_lex("a()<<1",
449
- s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
450
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
451
- :tLPAREN2, "(", :expr_beg, 1, 0,
452
- :tRPAREN, ")", :expr_endfn, 0, 0,
453
- :tLSHFT, "<<" , :expr_beg, 0, 0,
454
- :tINTEGER, 1, :expr_end, 0, 0)
455
- end
456
-
457
- def test_yylex_lambda_args__20
458
- setup_lexer_class RubyParser::V20
459
-
460
- assert_lex("-> (a) { }",
461
- s(:iter, s(:call, nil, :lambda),
462
- s(:args, :a)),
463
-
464
- :tLAMBDA, nil, :expr_endfn, 0, 0,
465
- :tLPAREN2, "(", :expr_beg, 1, 0,
466
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
467
- :tRPAREN, ")", :expr_endfn, 0, 0,
468
- :tLCURLY, "{", :expr_beg, 0, 1,
469
- :tRCURLY, "}", :expr_endarg, 0, 0)
470
- end
471
-
472
- def test_yylex_lambda_as_args_with_block__20
473
- setup_lexer_class RubyParser::V20
474
-
475
- assert_lex3("a -> do end do end",
476
- nil,
477
- :tIDENTIFIER, "a", :expr_cmdarg,
478
- :tLAMBDA, nil, :expr_endfn,
479
- :kDO, "do", :expr_beg,
480
- :kEND, "end", :expr_end,
481
- :kDO, "do", :expr_beg,
482
- :kEND, "end", :expr_end)
483
- end
484
-
485
- def test_yylex_lambda_args_opt__20
486
- setup_lexer_class RubyParser::V20
487
-
488
- assert_lex("-> (a=nil) { }",
489
- s(:iter, s(:call, nil, :lambda),
490
- s(:args, s(:lasgn, :a, s(:nil)))),
491
-
492
- :tLAMBDA, nil, :expr_endfn, 0, 0,
493
- :tLPAREN2, "(", :expr_beg, 1, 0,
494
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
495
- :tEQL, "=", :expr_beg, 1, 0,
496
- :kNIL, "nil", :expr_end, 1, 0,
497
- :tRPAREN, ")", :expr_endfn, 0, 0,
498
- :tLCURLY, "{", :expr_beg, 0, 1,
499
- :tRCURLY, "}", :expr_endarg, 0, 0)
500
- end
501
-
502
- def test_yylex_lambda_hash__20
503
- setup_lexer_class RubyParser::V20
504
-
505
- assert_lex("-> (a={}) { }",
506
- s(:iter, s(:call, nil, :lambda),
507
- s(:args, s(:lasgn, :a, s(:hash)))),
508
-
509
- :tLAMBDA, nil, :expr_endfn, 0, 0,
510
- :tLPAREN2, "(", :expr_beg, 1, 0,
511
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
512
- :tEQL, "=", :expr_beg, 1, 0,
513
- :tLBRACE, "{", :expr_beg, 1, 1,
514
- :tRCURLY, "}", :expr_endarg, 1, 0,
515
- :tRPAREN, ")", :expr_endfn, 0, 0,
516
- :tLCURLY, "{", :expr_beg, 0, 1,
517
- :tRCURLY, "}", :expr_endarg, 0, 0)
518
- end
519
-
520
- def test_yylex_iter_array_curly
521
- assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
522
- s(:iter,
523
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
524
- s(:args, :c, :d)),
525
-
526
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 0,
527
- :tSYMBOL, "a", :expr_end, 0, 0,
528
- :tCOMMA, ",", :expr_beg, 0, 0,
529
- :tLBRACK, "[", :expr_beg, 1, 0,
530
- :tSYMBOL, "b", :expr_end, 1, 0,
531
- :tRBRACK, "]", :expr_endarg, 0, 0,
532
- :tLBRACE_ARG, "{", :expr_beg, 0, 1,
533
- :tPIPE, "|", :expr_beg, 0, 1,
534
- :tIDENTIFIER, "c", :expr_arg, 0, 1,
535
- :tCOMMA, ",", :expr_beg, 0, 1,
536
- :tIDENTIFIER, "d", :expr_arg, 0, 1,
537
- :tPIPE, "|", :expr_beg, 0, 1,
538
- :tRCURLY, "}", :expr_endarg, 0, 0)
539
- end
540
-
541
- def test_yylex_const_call_same_name
542
- assert_lex("X = a { }; b { f :c }",
543
- s(:block,
544
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
545
- s(:iter,
546
- s(:call, nil, :b),
547
- 0,
548
- s(:call, nil, :f, s(:lit, :c)))),
549
-
550
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
551
- :tEQL, "=", :expr_beg, 0, 0,
552
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
553
- :tLCURLY, "{", :expr_beg, 0, 1,
554
- :tRCURLY, "}", :expr_endarg, 0, 0,
555
- :tSEMI, ";", :expr_beg, 0, 0,
556
-
557
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
558
- :tLCURLY, "{", :expr_beg, 0, 1,
559
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 1, # different
560
- :tSYMBOL, "c", :expr_end, 0, 1,
561
- :tRCURLY, "}", :expr_endarg, 0, 0)
562
-
563
- assert_lex("X = a { }; b { X :c }",
564
- s(:block,
565
- s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
566
- s(:iter,
567
- s(:call, nil, :b),
568
- 0,
569
- s(:call, nil, :X, s(:lit, :c)))),
570
-
571
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
572
- :tEQL, "=", :expr_beg, 0, 0,
573
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
574
- :tLCURLY, "{", :expr_beg, 0, 1,
575
- :tRCURLY, "}", :expr_endarg, 0, 0,
576
- :tSEMI, ";", :expr_beg, 0, 0,
577
-
578
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
579
- :tLCURLY, "{", :expr_beg, 0, 1,
580
- :tCONSTANT, "X", :expr_cmdarg, 0, 1, # same
581
- :tSYMBOL, "c", :expr_end, 0, 1,
582
- :tRCURLY, "}", :expr_endarg, 0, 0)
583
- end
584
-
585
- def test_yylex_lasgn_call_same_name
586
- assert_lex("a = b.c :d => 1",
587
- s(:lasgn, :a,
588
- s(:call, s(:call, nil, :b), :c,
589
- s(:hash, s(:lit, :d), s(:lit, 1)))),
590
-
591
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
592
- :tEQL, "=", :expr_beg, 0, 0,
593
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
594
- :tDOT, ".", :expr_dot, 0, 0,
595
- :tIDENTIFIER, "c", :expr_arg, 0, 0, # different
596
- :tSYMBOL, "d", :expr_end, 0, 0,
597
- :tASSOC, "=>", :expr_beg, 0, 0,
598
- :tINTEGER, 1, :expr_end, 0, 0)
599
-
600
- assert_lex("a = b.a :d => 1",
601
- s(:lasgn, :a,
602
- s(:call, s(:call, nil, :b), :a,
603
- s(:hash, s(:lit, :d), s(:lit, 1)))),
604
-
605
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
606
- :tEQL, "=", :expr_beg, 0, 0,
607
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
608
- :tDOT, ".", :expr_dot, 0, 0,
609
- :tIDENTIFIER, "a", :expr_arg, 0, 0, # same as lvar
610
- :tSYMBOL, "d", :expr_end, 0, 0,
611
- :tASSOC, "=>", :expr_beg, 0, 0,
612
- :tINTEGER, 1, :expr_end, 0, 0)
403
+ assert_lex3 "=>", nil, :tASSOC, "=>", EXPR_BEG
613
404
  end
614
405
 
615
406
  def test_yylex_back_ref
616
407
  assert_lex3("[$&, $`, $', $+]",
617
408
  nil,
618
- :tLBRACK, "[", :expr_beg,
619
- :tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
620
- :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
621
- :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
622
- :tBACK_REF, :+, :expr_end,
623
- :tRBRACK, "]", :expr_endarg)
409
+ :tLBRACK, "[", EXPR_PAR,
410
+ :tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
411
+ :tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
412
+ :tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
413
+ :tBACK_REF, :+, EXPR_END,
414
+ :tRBRACK, "]", EXPR_END)
624
415
  end
625
416
 
626
417
  def test_yylex_backslash
627
418
  assert_lex3("1 \\\n+ 2",
628
419
  nil,
629
- :tINTEGER, 1, :expr_end,
630
- :tPLUS, "+", :expr_beg,
631
- :tINTEGER, 2, :expr_end)
420
+ :tINTEGER, 1, EXPR_NUM,
421
+ :tPLUS, "+", EXPR_BEG,
422
+ :tINTEGER, 2, EXPR_NUM)
632
423
  end
633
424
 
634
425
  def test_yylex_backslash_bad
@@ -638,37 +429,37 @@ class TestRubyLexer < Minitest::Test
638
429
  def test_yylex_backtick
639
430
  assert_lex3("`ls`",
640
431
  nil,
641
- :tXSTRING_BEG, "`", :expr_beg,
642
- :tSTRING_CONTENT, "ls", :expr_beg,
643
- :tSTRING_END, "`", :expr_end)
432
+ :tXSTRING_BEG, "`", EXPR_BEG,
433
+ :tSTRING_CONTENT, "ls", EXPR_BEG,
434
+ :tSTRING_END, "`", EXPR_LIT)
644
435
  end
645
436
 
646
437
  def test_yylex_backtick_cmdarg
647
- self.lex_state = :expr_dot
438
+ self.lex_state = EXPR_DOT
648
439
 
649
440
  # \n ensures expr_cmd (TODO: why?)
650
- assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
441
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
651
442
  end
652
443
 
653
444
  def test_yylex_backtick_dot
654
- self.lex_state = :expr_dot
445
+ self.lex_state = EXPR_DOT
655
446
 
656
447
  assert_lex3("a.`(3)",
657
448
  nil,
658
- :tIDENTIFIER, "a", :expr_cmdarg,
659
- :tDOT, ".", :expr_dot,
660
- :tBACK_REF2, "`", :expr_arg,
661
- :tLPAREN2, "(", :expr_beg,
662
- :tINTEGER, 3, :expr_end,
663
- :tRPAREN, ")", :expr_endfn)
449
+ :tIDENTIFIER, "a", EXPR_CMDARG,
450
+ :tDOT, ".", EXPR_DOT,
451
+ :tBACK_REF2, "`", EXPR_ARG,
452
+ :tLPAREN2, "(", EXPR_PAR,
453
+ :tINTEGER, 3, EXPR_NUM,
454
+ :tRPAREN, ")", EXPR_ENDFN)
664
455
  end
665
456
 
666
457
  def test_yylex_backtick_method
667
- self.lex_state = :expr_fname
458
+ self.lex_state = EXPR_FNAME
668
459
 
669
460
  assert_lex3("`",
670
461
  nil,
671
- :tBACK_REF2, "`", :expr_end)
462
+ :tBACK_REF2, "`", EXPR_END)
672
463
  end
673
464
 
674
465
  def test_yylex_bad_char
@@ -676,24 +467,24 @@ class TestRubyLexer < Minitest::Test
676
467
  end
677
468
 
678
469
  def test_yylex_bang
679
- assert_lex3("!", nil, :tBANG, "!", :expr_beg)
470
+ assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
680
471
  end
681
472
 
682
473
  def test_yylex_bang_equals
683
- assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
474
+ assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
684
475
  end
685
476
 
686
477
  def test_yylex_bang_tilde
687
- assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
478
+ assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
688
479
  end
689
480
 
690
481
  def test_yylex_block_bug_1
691
482
  assert_lex3("a do end",
692
483
  s(:iter, s(:call, nil, :a), 0),
693
484
 
694
- :tIDENTIFIER, "a", :expr_cmdarg,
695
- :kDO, "do", :expr_beg,
696
- :kEND, "end", :expr_end)
485
+ :tIDENTIFIER, "a", EXPR_CMDARG,
486
+ :kDO, "do", EXPR_BEG,
487
+ :kEND, "end", EXPR_END)
697
488
  end
698
489
 
699
490
  def test_yylex_block_bug_2
@@ -702,70 +493,70 @@ class TestRubyLexer < Minitest::Test
702
493
  s(:lasgn, :a, s(:lit, 1)),
703
494
  s(:iter, s(:call, nil, :a), 0)),
704
495
 
705
- :tIDENTIFIER, "a", :expr_cmdarg,
706
- :tEQL, "=", :expr_beg,
707
- :tINTEGER, 1, :expr_end,
708
- :tNL, nil, :expr_beg,
709
- :tIDENTIFIER, "a", :expr_cmdarg,
710
- :kDO, "do", :expr_beg,
711
- :kEND, "end", :expr_end)
496
+ :tIDENTIFIER, "a", EXPR_CMDARG,
497
+ :tEQL, "=", EXPR_BEG,
498
+ :tINTEGER, 1, EXPR_NUM,
499
+ :tNL, nil, EXPR_BEG,
500
+ :tIDENTIFIER, "a", EXPR_CMDARG,
501
+ :kDO, "do", EXPR_BEG,
502
+ :kEND, "end", EXPR_END)
712
503
  end
713
504
 
714
505
  def test_yylex_block_bug_3
715
506
  assert_lex3("a { }",
716
507
  s(:iter, s(:call, nil, :a), 0),
717
508
 
718
- :tIDENTIFIER, "a", :expr_cmdarg, # verified
719
- :tLCURLY, "{", :expr_beg, # TODO: expr_beg|expr_label
720
- :tRCURLY, "}", :expr_endarg)
509
+ :tIDENTIFIER, "a", EXPR_CMDARG, # verified
510
+ :tLCURLY, "{", EXPR_PAR,
511
+ :tRCURLY, "}", EXPR_END)
721
512
  end
722
513
 
723
514
  def test_yylex_carat
724
- assert_lex3("^", nil, :tCARET, "^", :expr_beg)
515
+ assert_lex3("^", nil, :tCARET, "^", EXPR_BEG)
725
516
  end
726
517
 
727
518
  def test_yylex_carat_equals
728
- assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
519
+ assert_lex3("^=", nil, :tOP_ASGN, "^", EXPR_BEG)
729
520
  end
730
521
 
731
522
  def test_yylex_colon2
732
523
  assert_lex3("A::B",
733
524
  nil,
734
- :tCONSTANT, "A", :expr_cmdarg,
735
- :tCOLON2, "::", :expr_dot,
736
- :tCONSTANT, "B", :expr_arg)
525
+ :tCONSTANT, "A", EXPR_CMDARG,
526
+ :tCOLON2, "::", EXPR_DOT,
527
+ :tCONSTANT, "B", EXPR_ARG)
737
528
  end
738
529
 
739
530
  def test_yylex_colon2_argh
740
531
  assert_lex3("module X::Y\n c\nend",
741
532
  nil,
742
- :kMODULE, "module", :expr_beg,
743
- :tCONSTANT, "X", :expr_cmdarg,
744
- :tCOLON2, "::", :expr_dot,
745
- :tCONSTANT, "Y", :expr_arg,
746
- :tNL, nil, :expr_beg,
747
- :tIDENTIFIER, "c", :expr_cmdarg,
748
- :tNL, nil, :expr_beg,
749
- :kEND, "end", :expr_end)
533
+ :kMODULE, "module", EXPR_BEG,
534
+ :tCONSTANT, "X", EXPR_CMDARG,
535
+ :tCOLON2, "::", EXPR_DOT,
536
+ :tCONSTANT, "Y", EXPR_ARG,
537
+ :tNL, nil, EXPR_BEG,
538
+ :tIDENTIFIER, "c", EXPR_CMDARG,
539
+ :tNL, nil, EXPR_BEG,
540
+ :kEND, "end", EXPR_END)
750
541
  end
751
542
 
752
543
  def test_yylex_colon3
753
544
  assert_lex3("::Array",
754
545
  nil,
755
- :tCOLON3, "::", :expr_beg,
756
- :tCONSTANT, "Array", :expr_arg)
546
+ :tCOLON3, "::", EXPR_BEG,
547
+ :tCONSTANT, "Array", EXPR_ARG)
757
548
  end
758
549
 
759
550
  def test_yylex_comma
760
- assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
551
+ assert_lex3(",", nil, :tCOMMA, ",", EXPR_PAR)
761
552
  end
762
553
 
763
554
  def test_yylex_comment
764
555
  assert_lex3("1 # one\n# two\n2",
765
556
  nil,
766
- :tINTEGER, 1, :expr_end,
767
- :tNL, nil, :expr_beg,
768
- :tINTEGER, 2, :expr_end)
557
+ :tINTEGER, 1, EXPR_NUM,
558
+ :tNL, nil, EXPR_BEG,
559
+ :tINTEGER, 2, EXPR_NUM)
769
560
 
770
561
  assert_equal "# one\n# two\n", @lex.comments
771
562
  end
@@ -773,7 +564,7 @@ class TestRubyLexer < Minitest::Test
773
564
  def test_yylex_comment_begin
774
565
  assert_lex3("=begin\nblah\nblah\n=end\n42",
775
566
  nil,
776
- :tINTEGER, 42, :expr_end)
567
+ :tINTEGER, 42, EXPR_NUM)
777
568
 
778
569
  assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
779
570
  end
@@ -787,14 +578,14 @@ class TestRubyLexer < Minitest::Test
787
578
  def test_yylex_comment_begin_not_comment
788
579
  assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
789
580
  nil,
790
- :tIDENTIFIER, "beginfoo", :expr_cmdarg,
791
- :tEQL, "=", :expr_beg,
792
- :tINTEGER, 5, :expr_end,
793
- :tNL, nil, :expr_beg,
794
- :tIDENTIFIER, "p", :expr_cmdarg,
795
- :tIDENTIFIER, "x", :expr_arg,
796
- :tEQL, "=", :expr_beg,
797
- :tIDENTIFIER, "beginfoo", :expr_arg)
581
+ :tIDENTIFIER, "beginfoo", EXPR_CMDARG,
582
+ :tEQL, "=", EXPR_BEG,
583
+ :tINTEGER, 5, EXPR_NUM,
584
+ :tNL, nil, EXPR_BEG,
585
+ :tIDENTIFIER, "p", EXPR_CMDARG,
586
+ :tIDENTIFIER, "x", EXPR_ARG,
587
+ :tEQL, "=", EXPR_BEG,
588
+ :tIDENTIFIER, "beginfoo", EXPR_ARG)
798
589
  end
799
590
 
800
591
  def test_yylex_comment_begin_space
@@ -813,19 +604,63 @@ class TestRubyLexer < Minitest::Test
813
604
  assert_lex3("# comment", nil)
814
605
  end
815
606
 
816
- def test_yylex_constant
817
- assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
607
+ def test_yylex_const_call_same_name
608
+ assert_lex("X = a { }; b { f :c }",
609
+ s(:block,
610
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
611
+ s(:iter,
612
+ s(:call, nil, :b),
613
+ 0,
614
+ s(:call, nil, :f, s(:lit, :c)))),
615
+
616
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
617
+ :tEQL, "=", EXPR_BEG, 0, 0,
618
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
619
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
620
+ :tRCURLY, "}", EXPR_END, 0, 0,
621
+ :tSEMI, ";", EXPR_BEG, 0, 0,
622
+
623
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
624
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
625
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
626
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
627
+ :tRCURLY, "}", EXPR_END, 0, 0)
628
+
629
+ assert_lex("X = a { }; b { X :c }",
630
+ s(:block,
631
+ s(:cdecl, :X, s(:iter, s(:call, nil, :a), 0)),
632
+ s(:iter,
633
+ s(:call, nil, :b),
634
+ 0,
635
+ s(:call, nil, :X, s(:lit, :c)))),
636
+
637
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
638
+ :tEQL, "=", EXPR_BEG, 0, 0,
639
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
640
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
641
+ :tRCURLY, "}", EXPR_END, 0, 0,
642
+ :tSEMI, ";", EXPR_BEG, 0, 0,
643
+
644
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
645
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
646
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
647
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
648
+ :tRCURLY, "}", EXPR_END, 0, 0)
649
+ end
650
+
651
+ def test_yylex_constant
652
+ assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", EXPR_CMDARG)
818
653
  end
819
654
 
820
655
  def test_yylex_constant_semi
821
656
  assert_lex3("ArgumentError;",
822
657
  nil,
823
- :tCONSTANT, "ArgumentError", :expr_cmdarg,
824
- :tSEMI, ";", :expr_beg)
658
+ :tCONSTANT, "ArgumentError", EXPR_CMDARG,
659
+ :tSEMI, ";", EXPR_BEG)
825
660
  end
826
661
 
827
662
  def test_yylex_cvar
828
- assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
663
+ assert_lex3("@@blah", nil, :tCVAR, "@@blah", EXPR_END)
829
664
  end
830
665
 
831
666
  def test_yylex_cvar_bad
@@ -835,75 +670,67 @@ class TestRubyLexer < Minitest::Test
835
670
  end
836
671
 
837
672
  def test_yylex_def_bad_name
838
- self.lex_state = :expr_fname
673
+ self.lex_state = EXPR_FNAME
839
674
  refute_lex("def [ ", :kDEF, "def")
840
675
  end
841
676
 
842
677
  def test_yylex_div
843
678
  assert_lex3("a / 2",
844
679
  nil,
845
- :tIDENTIFIER, "a", :expr_cmdarg,
846
- :tDIVIDE, "/", :expr_beg,
847
- :tINTEGER, 2, :expr_end)
680
+ :tIDENTIFIER, "a", EXPR_CMDARG,
681
+ :tDIVIDE, "/", EXPR_BEG,
682
+ :tINTEGER, 2, EXPR_NUM)
848
683
  end
849
684
 
850
685
  def test_yylex_div_equals
851
686
  assert_lex3("a /= 2",
852
687
  nil,
853
- :tIDENTIFIER, "a", :expr_cmdarg,
854
- :tOP_ASGN, "/", :expr_beg,
855
- :tINTEGER, 2, :expr_end)
688
+ :tIDENTIFIER, "a", EXPR_CMDARG,
689
+ :tOP_ASGN, "/", EXPR_BEG,
690
+ :tINTEGER, 2, EXPR_NUM)
856
691
  end
857
692
 
858
693
  def test_yylex_do
859
694
  assert_lex3("x do 42 end",
860
695
  nil,
861
- :tIDENTIFIER, "x", :expr_cmdarg,
862
- :kDO, "do", :expr_beg,
863
- :tINTEGER, 42, :expr_end,
864
- :kEND, "end", :expr_end)
696
+ :tIDENTIFIER, "x", EXPR_CMDARG,
697
+ :kDO, "do", EXPR_BEG,
698
+ :tINTEGER, 42, EXPR_NUM,
699
+ :kEND, "end", EXPR_END)
865
700
  end
866
701
 
867
702
  def test_yylex_do_block
868
- self.lex_state = :expr_endarg
703
+ self.lex_state = EXPR_ENDARG
869
704
 
870
705
  assert_lex3("x.y do 42 end",
871
706
  nil,
872
- :tIDENTIFIER, "x", :expr_end,
873
- :tDOT, ".", :expr_dot,
874
- :tIDENTIFIER, "y", :expr_arg,
875
- :kDO_BLOCK, "do", :expr_beg,
876
- :tINTEGER, 42, :expr_end,
877
- :kEND, "end", :expr_end) do
707
+ :tIDENTIFIER, "x", EXPR_END,
708
+ :tDOT, ".", EXPR_DOT,
709
+ :tIDENTIFIER, "y", EXPR_ARG,
710
+ :kDO_BLOCK, "do", EXPR_BEG,
711
+ :tINTEGER, 42, EXPR_NUM,
712
+ :kEND, "end", EXPR_END) do
878
713
  @lex.cmdarg.push true
879
714
  end
880
715
  end
881
716
 
882
717
  def test_yylex_do_block2
883
- self.lex_state = :expr_endarg
718
+ self.lex_state = EXPR_ENDARG
884
719
 
885
720
  assert_lex3("do 42 end",
886
721
  nil,
887
- :kDO_BLOCK, "do", :expr_beg,
888
- :tINTEGER, 42, :expr_end,
889
- :kEND, "end", :expr_end)
890
- end
891
-
892
- def test_yylex_is_your_spacebar_broken?
893
- assert_lex3(":a!=:b",
894
- nil,
895
- :tSYMBOL, "a", :expr_end,
896
- :tNEQ, "!=", :expr_beg,
897
- :tSYMBOL, "b", :expr_end)
722
+ :kDO, "do", EXPR_BEG,
723
+ :tINTEGER, 42, EXPR_NUM,
724
+ :kEND, "end", EXPR_END)
898
725
  end
899
726
 
900
727
  def test_yylex_do_cond
901
728
  assert_lex3("x do 42 end",
902
729
  nil,
903
- :tIDENTIFIER, "x", :expr_cmdarg,
904
- :kDO_COND, "do", :expr_beg,
905
- :tINTEGER, 42, :expr_end,
906
- :kEND, "end", :expr_end) do
730
+ :tIDENTIFIER, "x", EXPR_CMDARG,
731
+ :kDO_COND, "do", EXPR_BEG,
732
+ :tINTEGER, 42, EXPR_NUM,
733
+ :kEND, "end", EXPR_END) do
907
734
  @lex.cond.push true
908
735
  end
909
736
  end
@@ -914,40 +741,40 @@ class TestRubyLexer < Minitest::Test
914
741
  end
915
742
 
916
743
  def test_yylex_dollar_eos
917
- assert_lex3("$", nil, "$", "$", :expr_end) # FIX: wtf is this?!?
744
+ assert_lex3("$", nil, "$", "$", EXPR_END) # FIX: wtf is this?!?
918
745
  end
919
746
 
920
747
  def test_yylex_dot # HINT message sends
921
- assert_lex3(".", nil, :tDOT, ".", :expr_dot)
748
+ assert_lex3(".", nil, :tDOT, ".", EXPR_DOT)
922
749
  end
923
750
 
924
751
  def test_yylex_dot2
925
- assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
752
+ assert_lex3("..", nil, :tDOT2, "..", EXPR_BEG)
926
753
  end
927
754
 
928
755
  def test_yylex_dot3
929
- assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
756
+ assert_lex3("...", nil, :tDOT3, "...", EXPR_BEG)
930
757
  end
931
758
 
932
759
  def test_yylex_equals
933
760
  # FIX: this sucks
934
- assert_lex3("=", nil, :tEQL, "=", :expr_beg)
761
+ assert_lex3("=", nil, :tEQL, "=", EXPR_BEG)
935
762
  end
936
763
 
937
764
  def test_yylex_equals2
938
- assert_lex3("==", nil, :tEQ, "==", :expr_beg)
765
+ assert_lex3("==", nil, :tEQ, "==", EXPR_BEG)
939
766
  end
940
767
 
941
768
  def test_yylex_equals3
942
- assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
769
+ assert_lex3("===", nil, :tEQQ, "===", EXPR_BEG)
943
770
  end
944
771
 
945
772
  def test_yylex_equals_tilde
946
- assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
773
+ assert_lex3("=~", nil, :tMATCH, "=~", EXPR_BEG)
947
774
  end
948
775
 
949
776
  def test_yylex_float
950
- assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
777
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
951
778
  end
952
779
 
953
780
  def test_yylex_float_bad_no_underscores
@@ -965,55 +792,48 @@ class TestRubyLexer < Minitest::Test
965
792
  def test_yylex_float_call
966
793
  assert_lex3("1.0.to_s",
967
794
  nil,
968
- :tFLOAT, 1.0, :expr_end,
969
- :tDOT, ".", :expr_dot,
970
- :tIDENTIFIER, "to_s", :expr_arg)
795
+ :tFLOAT, 1.0, EXPR_NUM,
796
+ :tDOT, ".", EXPR_DOT,
797
+ :tIDENTIFIER, "to_s", EXPR_ARG)
971
798
  end
972
799
 
973
800
  def test_yylex_float_dot_E
974
801
  assert_lex3("1.0E10",
975
802
  nil,
976
- :tFLOAT, 10000000000.0, :expr_end)
803
+ :tFLOAT, 10000000000.0, EXPR_NUM)
977
804
  end
978
805
 
979
806
  def test_yylex_float_dot_E_neg
980
807
  assert_lex3("-1.0E10",
981
808
  nil,
982
- :tUMINUS_NUM, "-", :expr_beg,
983
- :tFLOAT, 10000000000.0, :expr_end)
809
+ :tUMINUS_NUM, "-", EXPR_BEG,
810
+ :tFLOAT, 10000000000.0, EXPR_NUM)
984
811
  end
985
812
 
986
813
  def test_yylex_float_dot_e
987
814
  assert_lex3("1.0e10",
988
815
  nil,
989
- :tFLOAT, 10000000000.0, :expr_end)
816
+ :tFLOAT, 10000000000.0, EXPR_NUM)
990
817
  end
991
818
 
992
819
  def test_yylex_float_dot_e_neg
993
820
  assert_lex3("-1.0e10",
994
821
  nil,
995
- :tUMINUS_NUM, "-", :expr_beg,
996
- :tFLOAT, 10000000000.0, :expr_end)
822
+ :tUMINUS_NUM, "-", EXPR_BEG,
823
+ :tFLOAT, 10000000000.0, EXPR_NUM)
997
824
  end
998
825
 
999
826
  def test_yylex_float_e
1000
827
  assert_lex3("1e10",
1001
828
  nil,
1002
- :tFLOAT, 10000000000.0, :expr_end)
829
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1003
830
  end
1004
831
 
1005
832
  def test_yylex_float_e_bad_double_e
1006
833
  assert_lex3("1e2e3",
1007
834
  nil,
1008
- :tFLOAT, 100, :expr_end,
1009
- :tIDENTIFIER, "e3", :expr_end)
1010
- end
1011
-
1012
- def test_yylex_float_if_modifier
1013
- assert_lex3("1e2if",
1014
- nil,
1015
- :tFLOAT, 100, :expr_end,
1016
- :kIF_MOD, "if", :expr_beg)
835
+ :tFLOAT, 100, EXPR_NUM,
836
+ :tIDENTIFIER, "e3", EXPR_END)
1017
837
  end
1018
838
 
1019
839
  def test_yylex_float_e_bad_trailing_underscore
@@ -1021,198 +841,258 @@ class TestRubyLexer < Minitest::Test
1021
841
  end
1022
842
 
1023
843
  def test_yylex_float_e_minus
1024
- assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
844
+ assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, EXPR_NUM)
1025
845
  end
1026
846
 
1027
847
  def test_yylex_float_e_neg
1028
848
  assert_lex3("-1e10",
1029
849
  nil,
1030
- :tUMINUS_NUM, "-", :expr_beg,
1031
- :tFLOAT, 10000000000.0, :expr_end)
850
+ :tUMINUS_NUM, "-", EXPR_BEG,
851
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1032
852
  end
1033
853
 
1034
854
  def test_yylex_float_e_neg_minus
1035
855
  assert_lex3("-1e-10",
1036
856
  nil,
1037
- :tUMINUS_NUM, "-", :expr_beg,
1038
- :tFLOAT, 1.0e-10, :expr_end)
857
+ :tUMINUS_NUM, "-", EXPR_BEG,
858
+ :tFLOAT, 1.0e-10, EXPR_NUM)
1039
859
  end
1040
860
 
1041
861
  def test_yylex_float_e_neg_plus
1042
862
  assert_lex3("-1e+10",
1043
863
  nil,
1044
- :tUMINUS_NUM, "-", :expr_beg,
1045
- :tFLOAT, 10000000000.0, :expr_end)
864
+ :tUMINUS_NUM, "-", EXPR_BEG,
865
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1046
866
  end
1047
867
 
1048
868
  def test_yylex_float_e_plus
1049
- assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
869
+ assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, EXPR_NUM)
1050
870
  end
1051
871
 
1052
872
  def test_yylex_float_e_zero
1053
- assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
873
+ assert_lex3("0e0", nil, :tFLOAT, 0.0, EXPR_NUM)
874
+ end
875
+
876
+ def test_yylex_float_if_modifier
877
+ assert_lex3("1e2if",
878
+ nil,
879
+ :tFLOAT, 100, EXPR_NUM,
880
+ :kIF_MOD, "if", EXPR_PAR)
1054
881
  end
1055
882
 
1056
883
  def test_yylex_float_neg
1057
884
  assert_lex3("-1.0",
1058
885
  nil,
1059
- :tUMINUS_NUM, "-", :expr_beg,
1060
- :tFLOAT, 1.0, :expr_end)
886
+ :tUMINUS_NUM, "-", EXPR_BEG,
887
+ :tFLOAT, 1.0, EXPR_NUM)
1061
888
  end
1062
889
 
1063
890
  def test_yylex_ge
1064
891
  assert_lex3("a >= 2",
1065
892
  nil,
1066
- :tIDENTIFIER, "a", :expr_cmdarg,
1067
- :tGEQ, ">=", :expr_beg,
1068
- :tINTEGER, 2, :expr_end)
893
+ :tIDENTIFIER, "a", EXPR_CMDARG,
894
+ :tGEQ, ">=", EXPR_BEG,
895
+ :tINTEGER, 2, EXPR_NUM)
1069
896
  end
1070
897
 
1071
898
  def test_yylex_global
1072
- assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
899
+ assert_lex3("$blah", nil, :tGVAR, "$blah", EXPR_END)
1073
900
  end
1074
901
 
1075
902
  def test_yylex_global_backref
1076
- self.lex_state = :expr_fname
903
+ self.lex_state = EXPR_FNAME
1077
904
 
1078
- assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
905
+ assert_lex3("$`", nil, :tGVAR, "$`", EXPR_END)
1079
906
  end
1080
907
 
1081
908
  def test_yylex_global_dash_nothing
1082
- assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
909
+ assert_lex3("$- ", nil, :tGVAR, "$-", EXPR_END)
1083
910
  end
1084
911
 
1085
912
  def test_yylex_global_dash_something
1086
- assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
913
+ assert_lex3("$-x", nil, :tGVAR, "$-x", EXPR_END)
1087
914
  end
1088
915
 
1089
916
  def test_yylex_global_number
1090
- self.lex_state = :expr_fname
917
+ self.lex_state = EXPR_FNAME
1091
918
 
1092
- assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
919
+ assert_lex3("$1", nil, :tGVAR, "$1", EXPR_END)
1093
920
  end
1094
921
 
1095
922
  def test_yylex_global_number_big
1096
- self.lex_state = :expr_fname
923
+ self.lex_state = EXPR_FNAME
924
+
925
+ assert_lex3("$1234", nil, :tGVAR, "$1234", EXPR_END)
926
+ end
1097
927
 
1098
- assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
928
+ def test_yylex_global_I_have_no_words
929
+ assert_lex3("$x\xE2\x80\x8B = 42", # zero width space?!?!?
930
+ nil,
931
+ :tGVAR, "$x\xE2\x80\x8B", EXPR_END,
932
+ :tEQL, "=", EXPR_BEG,
933
+ :tINTEGER, 42, EXPR_NUM)
1099
934
  end
1100
935
 
1101
936
  def test_yylex_global_other
1102
937
  assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
1103
938
  nil,
1104
- :tLBRACK, "[", :expr_beg,
1105
- :tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
1106
- :tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
1107
- :tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
1108
- :tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
1109
- :tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
1110
- :tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
1111
- :tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
1112
- :tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
1113
- :tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
1114
- :tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
1115
- :tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
1116
- :tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
1117
- :tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
1118
- :tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
1119
- :tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
1120
- :tGVAR, "$\"", :expr_end,
1121
- :tRBRACK, "]", :expr_endarg)
939
+ :tLBRACK, "[", EXPR_PAR,
940
+ :tGVAR, "$~", EXPR_END, :tCOMMA, ",", EXPR_PAR,
941
+ :tGVAR, "$*", EXPR_END, :tCOMMA, ",", EXPR_PAR,
942
+ :tGVAR, "$$", EXPR_END, :tCOMMA, ",", EXPR_PAR,
943
+ :tGVAR, "$?", EXPR_END, :tCOMMA, ",", EXPR_PAR,
944
+ :tGVAR, "$!", EXPR_END, :tCOMMA, ",", EXPR_PAR,
945
+ :tGVAR, "$@", EXPR_END, :tCOMMA, ",", EXPR_PAR,
946
+ :tGVAR, "$/", EXPR_END, :tCOMMA, ",", EXPR_PAR,
947
+ :tGVAR, "$\\", EXPR_END, :tCOMMA, ",", EXPR_PAR,
948
+ :tGVAR, "$;", EXPR_END, :tCOMMA, ",", EXPR_PAR,
949
+ :tGVAR, "$,", EXPR_END, :tCOMMA, ",", EXPR_PAR,
950
+ :tGVAR, "$.", EXPR_END, :tCOMMA, ",", EXPR_PAR,
951
+ :tGVAR, "$=", EXPR_END, :tCOMMA, ",", EXPR_PAR,
952
+ :tGVAR, "$:", EXPR_END, :tCOMMA, ",", EXPR_PAR,
953
+ :tGVAR, "$<", EXPR_END, :tCOMMA, ",", EXPR_PAR,
954
+ :tGVAR, "$>", EXPR_END, :tCOMMA, ",", EXPR_PAR,
955
+ :tGVAR, "$\"", EXPR_END,
956
+ :tRBRACK, "]", EXPR_END)
1122
957
  end
1123
958
 
1124
959
  def test_yylex_global_underscore
1125
- assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
960
+ assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
1126
961
  end
1127
962
 
1128
963
  def test_yylex_global_wierd
1129
- assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
964
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
1130
965
  end
1131
966
 
1132
967
  def test_yylex_global_zero
1133
- assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
968
+ assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
1134
969
  end
1135
970
 
1136
971
  def test_yylex_gt
1137
972
  assert_lex3("a > 2",
1138
973
  nil,
1139
- :tIDENTIFIER, "a", :expr_cmdarg,
1140
- :tGT, ">", :expr_beg,
1141
- :tINTEGER, 2, :expr_end)
974
+ :tIDENTIFIER, "a", EXPR_CMDARG,
975
+ :tGT, ">", EXPR_BEG,
976
+ :tINTEGER, 2, EXPR_NUM)
1142
977
  end
1143
978
 
1144
- def test_yylex_heredoc_backtick
1145
- assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1146
- nil,
1147
- :tIDENTIFIER, "a", :expr_cmdarg,
1148
- :tEQL, "=", :expr_beg,
1149
- :tXSTRING_BEG, "`", :expr_beg,
1150
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1151
- :tSTRING_END, "EOF", :expr_end,
1152
- :tNL, nil, :expr_beg)
979
+ def test_yylex_hash_colon
980
+ assert_lex("{a:1}",
981
+ s(:hash, s(:lit, :a), s(:lit, 1)),
982
+
983
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
984
+ :tLABEL, "a", EXPR_LAB, 0, 1,
985
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
986
+ :tRCURLY, "}", EXPR_END, 0, 0)
1153
987
  end
1154
988
 
1155
- def test_yylex_heredoc_double
1156
- assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1157
- nil,
1158
- :tIDENTIFIER, "a", :expr_cmdarg,
1159
- :tEQL, "=", :expr_beg,
1160
- :tSTRING_BEG, "\"", :expr_beg,
1161
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1162
- :tSTRING_END, "EOF", :expr_end,
1163
- :tNL, nil, :expr_beg)
989
+ def test_yylex_hash_colon_double_quoted_symbol
990
+ assert_lex('{"abc": :b}',
991
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
992
+
993
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
994
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
995
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
996
+ :tRCURLY, "}", EXPR_END, 0, 0)
1164
997
  end
1165
998
 
1166
- def test_yylex_heredoc_double_dash
1167
- assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1168
- nil,
1169
- :tIDENTIFIER, "a", :expr_cmdarg,
1170
- :tEQL, "=", :expr_beg,
1171
- :tSTRING_BEG, "\"", :expr_beg,
1172
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1173
- :tSTRING_END, "EOF", :expr_end,
1174
- :tNL, nil, :expr_beg)
999
+ def test_yylex_hash_colon_double_quoted_symbol_22
1000
+ setup_lexer_class RubyParser::V22
1001
+
1002
+ assert_lex('{"abc": :b}',
1003
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1004
+
1005
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1006
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1007
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1008
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1175
1009
  end
1176
1010
 
1177
- def test_yylex_heredoc_double_squiggly
1178
- setup_lexer_class Ruby23Parser
1011
+ def test_yylex_hash_colon_double_quoted_with_escapes
1012
+ assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
1013
+ nil,
1179
1014
 
1180
- assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1181
- nil,
1182
- :tIDENTIFIER, "a", :expr_cmdarg,
1183
- :tEQL, "=", :expr_beg,
1184
- :tSTRING_BEG, "\"", :expr_beg,
1185
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1186
- :tSTRING_END, "EOF", :expr_end,
1187
- :tNL, nil, :expr_beg)
1015
+ :tLBRACE, "{", EXPR_PAR,
1016
+ :tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
1017
+ :tINTEGER, 1, EXPR_NUM,
1018
+ :tRCURLY, "}", EXPR_END)
1188
1019
  end
1189
1020
 
1190
- # mri handles tabs in a pretty specific way:
1191
- # https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
1192
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
1193
- setup_lexer_class Ruby23Parser
1021
+ def test_yylex_hash_colon_quoted_22
1022
+ setup_lexer_class RubyParser::V22
1194
1023
 
1195
- assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
1024
+ assert_lex("{'a':1}",
1025
+ s(:hash, s(:lit, :a), s(:lit, 1)),
1026
+
1027
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1028
+ :tLABEL, "a", EXPR_LAB, 0, 1,
1029
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
1030
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1031
+ end
1032
+
1033
+ def test_yylex_hash_colon_quoted_symbol
1034
+ assert_lex("{'abc': :b}",
1035
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1036
+
1037
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1038
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1039
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1040
+ :tRCURLY, "}", EXPR_END, 0, 0)
1041
+ end
1042
+
1043
+ def test_yylex_hash_colon_quoted_symbol_22
1044
+ setup_lexer_class RubyParser::V22
1045
+
1046
+ assert_lex("{'abc': :b}",
1047
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1048
+
1049
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1050
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1051
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1052
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1053
+ end
1054
+
1055
+ def test_yylex_hash_colon_quoted_with_escapes
1056
+ assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
1057
+ nil,
1058
+
1059
+ :tLBRACE, "{", EXPR_PAR,
1060
+ :tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
1061
+ :tINTEGER, 1, EXPR_NUM,
1062
+ :tRCURLY, "}", EXPR_END)
1063
+ end
1064
+
1065
+ def test_yylex_heredoc_backtick
1066
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1196
1067
  nil,
1197
- :tIDENTIFIER, "a", :expr_cmdarg,
1198
- :tEQL, "=", :expr_beg,
1199
- :tSTRING_BEG, "\"", :expr_beg,
1200
- :tSTRING_CONTENT, "blah blah\n\tblah blah\n", :expr_beg,
1201
- :tSTRING_END, "EOF", :expr_end,
1202
- :tNL, nil, :expr_beg)
1068
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1069
+ :tEQL, "=", EXPR_BEG,
1070
+ :tXSTRING_BEG, "`", EXPR_BEG,
1071
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1072
+ :tSTRING_END, "EOF", EXPR_LIT,
1073
+ :tNL, nil, EXPR_BEG)
1203
1074
  end
1204
1075
 
1205
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
1206
- setup_lexer_class Ruby23Parser
1076
+ def test_yylex_heredoc_double
1077
+ assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1078
+ nil,
1079
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1080
+ :tEQL, "=", EXPR_BEG,
1081
+ :tSTRING_BEG, "\"", EXPR_BEG,
1082
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1083
+ :tSTRING_END, "EOF", EXPR_LIT,
1084
+ :tNL, nil, EXPR_BEG)
1085
+ end
1207
1086
 
1208
- assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
1087
+ def test_yylex_heredoc_double_dash
1088
+ assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1209
1089
  nil,
1210
- :tIDENTIFIER, "a", :expr_cmdarg,
1211
- :tEQL, "=", :expr_beg,
1212
- :tSTRING_BEG, "\"", :expr_beg,
1213
- :tSTRING_CONTENT, "blah blah\n blah blah\n", :expr_beg,
1214
- :tSTRING_END, "EOF", :expr_end,
1215
- :tNL, nil, :expr_beg)
1090
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1091
+ :tEQL, "=", EXPR_BEG,
1092
+ :tSTRING_BEG, "\"", EXPR_BEG,
1093
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1094
+ :tSTRING_END, "EOF", EXPR_LIT,
1095
+ :tNL, nil, EXPR_BEG)
1216
1096
  end
1217
1097
 
1218
1098
  def test_yylex_heredoc_double_eos
@@ -1230,41 +1110,56 @@ class TestRubyLexer < Minitest::Test
1230
1110
  end
1231
1111
 
1232
1112
  def test_yylex_heredoc_double_interp
1233
- assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
1234
- nil,
1235
- :tIDENTIFIER, "a", :expr_cmdarg,
1236
- :tEQL, "=", :expr_beg,
1237
- :tSTRING_BEG, "\"", :expr_beg,
1238
- :tSTRING_CONTENT, "#x a ", :expr_beg,
1239
- :tSTRING_DVAR, "\#@", :expr_beg,
1240
- :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
1241
- :tSTRING_DVAR, "\#$", :expr_beg,
1242
- :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
1243
- :tSTRING_DBEG, "\#{", :expr_beg,
1244
- :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
1245
- :tSTRING_END, "EOF", :expr_end,
1246
- :tNL, nil, :expr_beg)
1113
+ assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#@@d \#{3} \nEOF\n\n",
1114
+ nil,
1115
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1116
+ :tEQL, "=", EXPR_BEG,
1117
+ :tSTRING_BEG, "\"", EXPR_BEG,
1118
+ :tSTRING_CONTENT, "#x a ", EXPR_BEG,
1119
+ :tSTRING_DVAR, "#", EXPR_BEG,
1120
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG, # HUH?
1121
+ :tSTRING_DVAR, "#", EXPR_BEG,
1122
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG, # HUH?
1123
+ :tSTRING_DVAR, "#", EXPR_BEG,
1124
+ :tSTRING_CONTENT, "@@d ", EXPR_BEG, # HUH?
1125
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1126
+ :tSTRING_CONTENT, "3} \n", EXPR_BEG,
1127
+ :tSTRING_END, "EOF", EXPR_LIT,
1128
+ :tNL, nil, EXPR_BEG)
1129
+ end
1130
+
1131
+ def test_yylex_heredoc_double_squiggly
1132
+ setup_lexer_class Ruby23Parser
1133
+
1134
+ assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1135
+ nil,
1136
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1137
+ :tEQL, "=", EXPR_BEG,
1138
+ :tSTRING_BEG, "\"", EXPR_BEG,
1139
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1140
+ :tSTRING_END, "EOF", EXPR_LIT,
1141
+ :tNL, nil, EXPR_BEG)
1247
1142
  end
1248
1143
 
1249
1144
  def test_yylex_heredoc_empty
1250
1145
  assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1251
1146
  nil,
1252
- :tSTRING_BEG, "\"", :expr_beg,
1253
- :tSTRING_DBEG, "\#{", :expr_beg,
1254
- :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
1255
- :tSTRING_END, "", :expr_end,
1256
- :tNL, nil, :expr_beg)
1147
+ :tSTRING_BEG, "\"", EXPR_BEG,
1148
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1149
+ :tSTRING_CONTENT, "x}\nblah2\n", EXPR_BEG,
1150
+ :tSTRING_END, "", EXPR_LIT,
1151
+ :tNL, nil, EXPR_BEG)
1257
1152
  end
1258
1153
 
1259
1154
  def test_yylex_heredoc_none
1260
1155
  assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
1261
1156
  nil,
1262
- :tIDENTIFIER, "a", :expr_cmdarg,
1263
- :tEQL, "=", :expr_beg,
1264
- :tSTRING_BEG, "\"", :expr_beg,
1265
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1266
- :tSTRING_END, "EOF", :expr_end,
1267
- :tNL, nil, :expr_beg)
1157
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1158
+ :tEQL, "=", EXPR_BEG,
1159
+ :tSTRING_BEG, "\"", EXPR_BEG,
1160
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1161
+ :tSTRING_END, "EOF", EXPR_LIT,
1162
+ :tNL, nil, EXPR_BEG)
1268
1163
  end
1269
1164
 
1270
1165
  def test_yylex_heredoc_none_bad_eos
@@ -1277,12 +1172,12 @@ class TestRubyLexer < Minitest::Test
1277
1172
  def test_yylex_heredoc_none_dash
1278
1173
  assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
1279
1174
  nil,
1280
- :tIDENTIFIER, "a", :expr_cmdarg,
1281
- :tEQL, "=", :expr_beg,
1282
- :tSTRING_BEG, "\"", :expr_beg,
1283
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1284
- :tSTRING_END, "EOF", :expr_end,
1285
- :tNL, nil, :expr_beg)
1175
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1176
+ :tEQL, "=", EXPR_BEG,
1177
+ :tSTRING_BEG, "\"", EXPR_BEG,
1178
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1179
+ :tSTRING_END, "EOF", EXPR_LIT,
1180
+ :tNL, nil, EXPR_BEG)
1286
1181
  end
1287
1182
 
1288
1183
  def test_yylex_heredoc_none_squiggly
@@ -1290,23 +1185,23 @@ class TestRubyLexer < Minitest::Test
1290
1185
 
1291
1186
  assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
1292
1187
  nil,
1293
- :tIDENTIFIER, "a", :expr_cmdarg,
1294
- :tEQL, "=", :expr_beg,
1295
- :tSTRING_BEG, "\"", :expr_beg,
1296
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1297
- :tSTRING_END, "EOF", :expr_end,
1298
- :tNL, nil, :expr_beg)
1188
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1189
+ :tEQL, "=", EXPR_BEG,
1190
+ :tSTRING_BEG, "\"", EXPR_BEG,
1191
+ :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1192
+ :tSTRING_END, "EOF", EXPR_LIT,
1193
+ :tNL, nil, EXPR_BEG)
1299
1194
  end
1300
1195
 
1301
1196
  def test_yylex_heredoc_single
1302
1197
  assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1303
1198
  nil,
1304
- :tIDENTIFIER, "a", :expr_cmdarg,
1305
- :tEQL, "=", :expr_beg,
1306
- :tSTRING_BEG, "\"", :expr_beg,
1307
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1308
- :tSTRING_END, "EOF", :expr_end,
1309
- :tNL, nil, :expr_beg)
1199
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1200
+ :tEQL, "=", EXPR_BEG,
1201
+ :tSTRING_BEG, "\"", EXPR_BEG,
1202
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1203
+ :tSTRING_END, "EOF", EXPR_LIT,
1204
+ :tNL, nil, EXPR_BEG)
1310
1205
  end
1311
1206
 
1312
1207
  def test_yylex_heredoc_single_bad_eos_body
@@ -1340,12 +1235,12 @@ class TestRubyLexer < Minitest::Test
1340
1235
  def test_yylex_heredoc_single_dash
1341
1236
  assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
1342
1237
  nil,
1343
- :tIDENTIFIER, "a", :expr_cmdarg,
1344
- :tEQL, "=", :expr_beg,
1345
- :tSTRING_BEG, "\"", :expr_beg,
1346
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1347
- :tSTRING_END, "EOF", :expr_end,
1348
- :tNL, nil, :expr_beg)
1238
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1239
+ :tEQL, "=", EXPR_BEG,
1240
+ :tSTRING_BEG, "\"", EXPR_BEG,
1241
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1242
+ :tSTRING_END, "EOF", EXPR_LIT,
1243
+ :tNL, nil, EXPR_BEG)
1349
1244
  end
1350
1245
 
1351
1246
  def test_yylex_heredoc_single_squiggly
@@ -1353,24 +1248,24 @@ class TestRubyLexer < Minitest::Test
1353
1248
 
1354
1249
  assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
1355
1250
  nil,
1356
- :tIDENTIFIER, "a", :expr_cmdarg,
1357
- :tEQL, "=", :expr_beg,
1358
- :tSTRING_BEG, "\"", :expr_beg,
1359
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1360
- :tSTRING_END, "EOF", :expr_end,
1361
- :tNL, nil, :expr_beg)
1251
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1252
+ :tEQL, "=", EXPR_BEG,
1253
+ :tSTRING_BEG, "\"", EXPR_BEG,
1254
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1255
+ :tSTRING_END, "EOF", EXPR_LIT,
1256
+ :tNL, nil, EXPR_BEG)
1362
1257
  end
1363
1258
 
1364
1259
  def test_yylex_identifier
1365
1260
  assert_lex3("identifier",
1366
1261
  nil,
1367
- :tIDENTIFIER, "identifier", :expr_cmdarg)
1262
+ :tIDENTIFIER, "identifier", EXPR_CMDARG)
1368
1263
  end
1369
1264
 
1370
1265
  def test_yylex_identifier_bang
1371
1266
  assert_lex3("identifier!",
1372
1267
  nil,
1373
- :tFID, "identifier!", :expr_cmdarg)
1268
+ :tFID, "identifier!", EXPR_CMDARG)
1374
1269
  end
1375
1270
 
1376
1271
  def test_yylex_identifier_cmp
@@ -1380,57 +1275,57 @@ class TestRubyLexer < Minitest::Test
1380
1275
  def test_yylex_identifier_def__20
1381
1276
  setup_lexer_class RubyParser::V20
1382
1277
 
1383
- assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1278
+ assert_lex_fname "identifier", :tIDENTIFIER, EXPR_ENDFN
1384
1279
  end
1385
1280
 
1386
1281
  def test_yylex_identifier_eh
1387
- assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
1388
- end
1389
-
1390
- def test_yylex_identifier_equals_arrow
1391
- assert_lex3(":blah==>",
1392
- nil,
1393
- :tSYMBOL, "blah=", :expr_end,
1394
- :tASSOC, "=>", :expr_beg)
1282
+ assert_lex3("identifier?", nil, :tFID, "identifier?", EXPR_CMDARG)
1395
1283
  end
1396
1284
 
1397
1285
  def test_yylex_identifier_equals3
1398
1286
  assert_lex3(":a===b",
1399
1287
  nil,
1400
- :tSYMBOL, "a", :expr_end,
1401
- :tEQQ, "===", :expr_beg,
1402
- :tIDENTIFIER, "b", :expr_arg)
1288
+ :tSYMBOL, "a", EXPR_LIT,
1289
+ :tEQQ, "===", EXPR_BEG,
1290
+ :tIDENTIFIER, "b", EXPR_ARG)
1403
1291
  end
1404
1292
 
1405
- def test_yylex_identifier_equals_equals_arrow
1406
- assert_lex3(":a==>b",
1293
+ def test_yylex_identifier_equals_arrow
1294
+ assert_lex3(":blah==>",
1407
1295
  nil,
1408
- :tSYMBOL, "a=", :expr_end,
1409
- :tASSOC, "=>", :expr_beg,
1410
- :tIDENTIFIER, "b", :expr_arg)
1296
+ :tSYMBOL, "blah=", EXPR_LIT,
1297
+ :tASSOC, "=>", EXPR_BEG)
1411
1298
  end
1412
1299
 
1413
1300
  def test_yylex_identifier_equals_caret
1414
1301
  assert_lex_fname "^", :tCARET
1415
1302
  end
1416
1303
 
1304
+ def test_yylex_identifier_equals_def2
1305
+ assert_lex_fname "==", :tEQ
1306
+ end
1307
+
1417
1308
  def test_yylex_identifier_equals_def__20
1418
1309
  setup_lexer_class RubyParser::V20
1419
1310
 
1420
- assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1311
+ assert_lex_fname "identifier=", :tIDENTIFIER, EXPR_ENDFN
1421
1312
  end
1422
1313
 
1423
- def test_yylex_identifier_equals_def2
1424
- assert_lex_fname "==", :tEQ
1314
+ def test_yylex_identifier_equals_equals_arrow
1315
+ assert_lex3(":a==>b",
1316
+ nil,
1317
+ :tSYMBOL, "a=", EXPR_LIT,
1318
+ :tASSOC, "=>", EXPR_BEG,
1319
+ :tIDENTIFIER, "b", EXPR_ARG)
1425
1320
  end
1426
1321
 
1427
1322
  def test_yylex_identifier_equals_expr
1428
- self.lex_state = :expr_dot
1323
+ self.lex_state = EXPR_DOT
1429
1324
  assert_lex3("y = arg",
1430
1325
  nil,
1431
- :tIDENTIFIER, "y", :expr_cmdarg,
1432
- :tEQL, "=", :expr_beg,
1433
- :tIDENTIFIER, "arg", :expr_arg)
1326
+ :tIDENTIFIER, "y", EXPR_CMDARG,
1327
+ :tEQL, "=", EXPR_BEG,
1328
+ :tIDENTIFIER, "arg", EXPR_ARG)
1434
1329
  end
1435
1330
 
1436
1331
  def test_yylex_identifier_equals_or
@@ -1442,12 +1337,12 @@ class TestRubyLexer < Minitest::Test
1442
1337
  end
1443
1338
 
1444
1339
  def test_yylex_identifier_equals_tilde
1445
- self.lex_state = :expr_fname # can only set via parser's defs
1340
+ self.lex_state = EXPR_FNAME # can only set via parser's defs
1446
1341
 
1447
1342
  assert_lex3("identifier=~",
1448
1343
  nil,
1449
- :tIDENTIFIER, "identifier", :expr_endfn,
1450
- :tMATCH, "=~", :expr_beg)
1344
+ :tIDENTIFIER, "identifier", EXPR_ENDFN,
1345
+ :tMATCH, "=~", EXPR_BEG)
1451
1346
  end
1452
1347
 
1453
1348
  def test_yylex_identifier_gt
@@ -1475,11 +1370,11 @@ class TestRubyLexer < Minitest::Test
1475
1370
  end
1476
1371
 
1477
1372
  def test_yylex_integer
1478
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1373
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1479
1374
  end
1480
1375
 
1481
1376
  def test_yylex_integer_bin
1482
- assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
1377
+ assert_lex3("0b101010", nil, :tINTEGER, 42, EXPR_NUM)
1483
1378
  end
1484
1379
 
1485
1380
  def test_yylex_integer_bin_bad_none
@@ -1491,7 +1386,7 @@ class TestRubyLexer < Minitest::Test
1491
1386
  end
1492
1387
 
1493
1388
  def test_yylex_integer_dec
1494
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1389
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1495
1390
  end
1496
1391
 
1497
1392
  def test_yylex_integer_dec_bad_underscores
@@ -1499,7 +1394,7 @@ class TestRubyLexer < Minitest::Test
1499
1394
  end
1500
1395
 
1501
1396
  def test_yylex_integer_dec_d
1502
- assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
1397
+ assert_lex3("0d42", nil, :tINTEGER, 42, EXPR_NUM)
1503
1398
  end
1504
1399
 
1505
1400
  def test_yylex_integer_dec_d_bad_none
@@ -1510,27 +1405,8 @@ class TestRubyLexer < Minitest::Test
1510
1405
  refute_lex "0d42__24"
1511
1406
  end
1512
1407
 
1513
- def test_yylex_integer_if_modifier
1514
- assert_lex3("123if",
1515
- nil,
1516
- :tINTEGER, 123, :expr_end,
1517
- :kIF_MOD, "if", :expr_beg)
1518
- end
1519
-
1520
- def test_yylex_question_eh_a__20
1521
- setup_lexer_class RubyParser::V20
1522
-
1523
- assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
1524
- end
1525
-
1526
- def test_yylex_question_eh_escape_M_escape_C__20
1527
- setup_lexer_class RubyParser::V20
1528
-
1529
- assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
1530
- end
1531
-
1532
1408
  def test_yylex_integer_hex
1533
- assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
1409
+ assert_lex3 "0x2a", nil, :tINTEGER, 42, EXPR_NUM
1534
1410
  end
1535
1411
 
1536
1412
  def test_yylex_integer_hex_bad_none
@@ -1541,24 +1417,19 @@ class TestRubyLexer < Minitest::Test
1541
1417
  refute_lex "0xab__cd"
1542
1418
  end
1543
1419
 
1544
- def test_yylex_integer_oct
1545
- assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
1546
- end
1547
-
1548
- def test_yylex_integer_oct_bad_range
1549
- refute_lex "08"
1550
- end
1551
-
1552
- def test_yylex_integer_oct_bad_range2
1553
- refute_lex "08"
1420
+ def test_yylex_integer_if_modifier
1421
+ assert_lex3("123if",
1422
+ nil,
1423
+ :tINTEGER, 123, EXPR_NUM,
1424
+ :kIF_MOD, "if", EXPR_PAR)
1554
1425
  end
1555
1426
 
1556
- def test_yylex_integer_oct_bad_underscores
1557
- refute_lex "01__23"
1427
+ def test_yylex_integer_oct
1428
+ assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
1558
1429
  end
1559
1430
 
1560
1431
  def test_yylex_integer_oct_O
1561
- assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
1432
+ assert_lex3 "0O52", nil, :tINTEGER, 42, EXPR_NUM
1562
1433
  end
1563
1434
 
1564
1435
  def test_yylex_integer_oct_O_bad_range
@@ -1570,11 +1441,23 @@ class TestRubyLexer < Minitest::Test
1570
1441
  end
1571
1442
 
1572
1443
  def test_yylex_integer_oct_O_not_bad_none
1573
- assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
1444
+ assert_lex3 "0O ", nil, :tINTEGER, 0, EXPR_NUM
1445
+ end
1446
+
1447
+ def test_yylex_integer_oct_bad_range
1448
+ refute_lex "08"
1449
+ end
1450
+
1451
+ def test_yylex_integer_oct_bad_range2
1452
+ refute_lex "08"
1453
+ end
1454
+
1455
+ def test_yylex_integer_oct_bad_underscores
1456
+ refute_lex "01__23"
1574
1457
  end
1575
1458
 
1576
1459
  def test_yylex_integer_oct_o
1577
- assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
1460
+ assert_lex3 "0o52", nil, :tINTEGER, 42, EXPR_NUM
1578
1461
  end
1579
1462
 
1580
1463
  def test_yylex_integer_oct_o_bad_range
@@ -1586,19 +1469,19 @@ class TestRubyLexer < Minitest::Test
1586
1469
  end
1587
1470
 
1588
1471
  def test_yylex_integer_oct_o_not_bad_none
1589
- assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
1472
+ assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
1590
1473
  end
1591
1474
 
1592
1475
  def test_yylex_integer_trailing
1593
1476
  assert_lex3("1.to_s",
1594
1477
  nil,
1595
- :tINTEGER, 1, :expr_end,
1596
- :tDOT, ".", :expr_dot,
1597
- :tIDENTIFIER, "to_s", :expr_arg)
1478
+ :tINTEGER, 1, EXPR_NUM,
1479
+ :tDOT, ".", EXPR_DOT,
1480
+ :tIDENTIFIER, "to_s", EXPR_ARG)
1598
1481
  end
1599
1482
 
1600
1483
  def test_yylex_integer_underscore
1601
- assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
1484
+ assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
1602
1485
  end
1603
1486
 
1604
1487
  def test_yylex_integer_underscore_bad
@@ -1606,11 +1489,62 @@ class TestRubyLexer < Minitest::Test
1606
1489
  end
1607
1490
 
1608
1491
  def test_yylex_integer_zero
1609
- assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
1492
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1493
+ end
1494
+
1495
+ def test_yylex_is_your_spacebar_broken?
1496
+ assert_lex3(":a!=:b",
1497
+ nil,
1498
+ :tSYMBOL, "a", EXPR_LIT,
1499
+ :tNEQ, "!=", EXPR_BEG,
1500
+ :tSYMBOL, "b", EXPR_LIT)
1501
+ end
1502
+
1503
+ def test_yylex_iter_array_curly
1504
+ # this will lex, but doesn't parse... don't freak out.
1505
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1506
+ nil,
1507
+
1508
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1509
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1510
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1511
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1512
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1513
+ :tRBRACK, "]", EXPR_END, 0, 0,
1514
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1515
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1516
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1517
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1518
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1519
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1520
+ :tRCURLY, "}", EXPR_END, 0, 0)
1521
+ end
1522
+
1523
+ def test_yylex_iter_array_curly__24
1524
+ setup_lexer_class RubyParser::V24
1525
+
1526
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1527
+ s(:iter,
1528
+ s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
1529
+ s(:args, :c, :d)),
1530
+
1531
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1532
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1533
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1534
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1535
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1536
+ :tRBRACK, "]", EXPR_ENDARG, 0, 0,
1537
+ :tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
1538
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1539
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1540
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1541
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1542
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1543
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1610
1544
  end
1611
1545
 
1612
1546
  def test_yylex_ivar
1613
- assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
1547
+ assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
1614
1548
  end
1615
1549
 
1616
1550
  def test_yylex_ivar_bad
@@ -1618,291 +1552,590 @@ class TestRubyLexer < Minitest::Test
1618
1552
  end
1619
1553
 
1620
1554
  def test_yylex_ivar_bad_0_length
1621
- refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
1555
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
1622
1556
  end
1623
1557
 
1624
1558
  def test_yylex_keyword_expr
1625
- self.lex_state = :expr_endarg
1559
+ self.lex_state = EXPR_ENDARG
1560
+
1561
+ assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
1562
+ end
1626
1563
 
1627
- assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
1564
+ def test_yylex_label
1565
+ assert_lex3("{a:",
1566
+ nil,
1567
+ :tLBRACE, "{", EXPR_PAR,
1568
+ :tLABEL, "a", EXPR_LAB)
1569
+ end
1570
+
1571
+ def test_yylex_label_in_params
1572
+ assert_lex3("foo(a:",
1573
+ nil,
1574
+ :tIDENTIFIER, "foo", EXPR_CMDARG,
1575
+ :tLPAREN2, "(", EXPR_PAR,
1576
+ :tLABEL, "a", EXPR_LAB)
1577
+ end
1578
+
1579
+ def test_yylex_lambda_args
1580
+ assert_lex("-> (a) { }",
1581
+ s(:iter, s(:lambda),
1582
+ s(:args, :a)),
1583
+
1584
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1585
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1586
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1587
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1588
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1589
+ :tRCURLY, "}", EXPR_END, 0, 0)
1590
+ end
1591
+
1592
+ def test_yylex_lambda_args__24
1593
+ setup_lexer_class RubyParser::V24
1594
+
1595
+ assert_lex("-> (a) { }",
1596
+ s(:iter, s(:lambda),
1597
+ s(:args, :a)),
1598
+
1599
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1600
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1601
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1602
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1603
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1604
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1605
+ end
1606
+
1607
+ def test_yylex_lambda_args_opt
1608
+ assert_lex("-> (a=nil) { }",
1609
+ s(:iter, s(:lambda),
1610
+ s(:args, s(:lasgn, :a, s(:nil)))),
1611
+
1612
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1613
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1614
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1615
+ :tEQL, "=", EXPR_BEG, 1, 0,
1616
+ :kNIL, "nil", EXPR_END, 1, 0,
1617
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1618
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1619
+ :tRCURLY, "}", EXPR_END, 0, 0)
1620
+ end
1621
+
1622
+ def test_yylex_lambda_args_opt__24
1623
+ setup_lexer_class RubyParser::V24
1624
+
1625
+ assert_lex("-> (a=nil) { }",
1626
+ s(:iter, s(:lambda),
1627
+ s(:args, s(:lasgn, :a, s(:nil)))),
1628
+
1629
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1630
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1631
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1632
+ :tEQL, "=", EXPR_BEG, 1, 0,
1633
+ :kNIL, "nil", EXPR_END, 1, 0,
1634
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1635
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1636
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1637
+ end
1638
+
1639
+ def test_yylex_lambda_as_args_with_block
1640
+ assert_lex3("a -> do end do end",
1641
+ nil,
1642
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1643
+ :tLAMBDA, nil, EXPR_ENDFN,
1644
+ :kDO, "do", EXPR_BEG,
1645
+ :kEND, "end", EXPR_END,
1646
+ :kDO, "do", EXPR_BEG,
1647
+ :kEND, "end", EXPR_END)
1648
+ end
1649
+
1650
+ def test_yylex_lambda_hash
1651
+ assert_lex("-> (a={}) { }",
1652
+ s(:iter, s(:lambda),
1653
+ s(:args, s(:lasgn, :a, s(:hash)))),
1654
+
1655
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1656
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1657
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1658
+ :tEQL, "=", EXPR_BEG, 1, 0,
1659
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1660
+ :tRCURLY, "}", EXPR_END, 1, 0,
1661
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1662
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1663
+ :tRCURLY, "}", EXPR_END, 0, 0)
1664
+ end
1665
+
1666
+ def test_yylex_lambda_hash__24
1667
+ setup_lexer_class RubyParser::V24
1668
+
1669
+ assert_lex("-> (a={}) { }",
1670
+ s(:iter, s(:lambda),
1671
+ s(:args, s(:lasgn, :a, s(:hash)))),
1672
+
1673
+ :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1674
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1675
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1676
+ :tEQL, "=", EXPR_BEG, 1, 0,
1677
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1678
+ :tRCURLY, "}", EXPR_ENDARG, 1, 0,
1679
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1680
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1681
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1682
+ end
1683
+
1684
+ def test_yylex_lasgn_call_same_name
1685
+ assert_lex("a = b.c :d => 1",
1686
+ s(:lasgn, :a,
1687
+ s(:call, s(:call, nil, :b), :c,
1688
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1689
+
1690
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1691
+ :tEQL, "=", EXPR_BEG, 0, 0,
1692
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1693
+ :tDOT, ".", EXPR_DOT, 0, 0,
1694
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
1695
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1696
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1697
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1698
+
1699
+ assert_lex("a = b.a :d => 1",
1700
+ s(:lasgn, :a,
1701
+ s(:call, s(:call, nil, :b), :a,
1702
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1703
+
1704
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1705
+ :tEQL, "=", EXPR_BEG, 0, 0,
1706
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1707
+ :tDOT, ".", EXPR_DOT, 0, 0,
1708
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
1709
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1710
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1711
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1628
1712
  end
1629
1713
 
1630
1714
  def test_yylex_lt
1631
- assert_lex3("<", nil, :tLT, "<", :expr_beg)
1715
+ assert_lex3("<", nil, :tLT, "<", EXPR_BEG)
1632
1716
  end
1633
1717
 
1634
1718
  def test_yylex_lt2
1635
1719
  assert_lex3("a << b",
1636
1720
  nil,
1637
- :tIDENTIFIER, "a", :expr_cmdarg,
1638
- :tLSHFT, "<<", :expr_beg,
1639
- :tIDENTIFIER, "b", :expr_arg)
1721
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1722
+ :tLSHFT, "<<", EXPR_BEG,
1723
+ :tIDENTIFIER, "b", EXPR_ARG)
1640
1724
  end
1641
1725
 
1642
1726
  def test_yylex_lt2_equals
1643
1727
  assert_lex3("a <<= b",
1644
1728
  nil,
1645
- :tIDENTIFIER, "a", :expr_cmdarg,
1646
- :tOP_ASGN, "<<", :expr_beg,
1647
- :tIDENTIFIER, "b", :expr_arg)
1729
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1730
+ :tOP_ASGN, "<<", EXPR_BEG,
1731
+ :tIDENTIFIER, "b", EXPR_ARG)
1648
1732
  end
1649
1733
 
1650
1734
  def test_yylex_lt_equals
1651
- assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
1735
+ assert_lex3("<=", nil, :tLEQ, "<=", EXPR_BEG)
1736
+ end
1737
+
1738
+ def test_yylex_method_parens_chevron
1739
+ assert_lex("a()<<1",
1740
+ s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
1741
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1742
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1743
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1744
+ :tLSHFT, "<<", EXPR_BEG, 0, 0,
1745
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1652
1746
  end
1653
1747
 
1654
1748
  def test_yylex_minus
1655
1749
  assert_lex3("1 - 2",
1656
1750
  nil,
1657
- :tINTEGER, 1, :expr_end,
1658
- :tMINUS, "-", :expr_beg,
1659
- :tINTEGER, 2, :expr_end)
1751
+ :tINTEGER, 1, EXPR_NUM,
1752
+ :tMINUS, "-", EXPR_BEG,
1753
+ :tINTEGER, 2, EXPR_NUM)
1660
1754
  end
1661
1755
 
1662
1756
  def test_yylex_minus_equals
1663
- assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
1757
+ assert_lex3("-=", nil, :tOP_ASGN, "-", EXPR_BEG)
1664
1758
  end
1665
1759
 
1666
1760
  def test_yylex_minus_method
1667
- self.lex_state = :expr_fname
1761
+ self.lex_state = EXPR_FNAME
1668
1762
 
1669
- assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
1763
+ assert_lex3("-", nil, :tMINUS, "-", EXPR_ARG)
1670
1764
  end
1671
1765
 
1672
1766
  def test_yylex_minus_unary_method
1673
- self.lex_state = :expr_fname
1767
+ self.lex_state = EXPR_FNAME
1674
1768
 
1675
- assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
1769
+ assert_lex3("-@", nil, :tUMINUS, "-@", EXPR_ARG)
1676
1770
  end
1677
1771
 
1678
1772
  def test_yylex_minus_unary_number
1679
1773
  assert_lex3("-42",
1680
1774
  nil,
1681
- :tUMINUS_NUM, "-", :expr_beg,
1682
- :tINTEGER, 42, :expr_end)
1775
+ :tUMINUS_NUM, "-", EXPR_BEG,
1776
+ :tINTEGER, 42, EXPR_NUM)
1777
+ end
1778
+
1779
+ def test_yylex_not_at_defn
1780
+ assert_lex("def +@; end",
1781
+ s(:defn, :+@, s(:args), s(:nil)),
1782
+
1783
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1784
+ :tUPLUS, "+@", EXPR_ARG, 0, 0,
1785
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1786
+ :kEND, "end", EXPR_END, 0, 0)
1787
+
1788
+ assert_lex("def !@; end",
1789
+ s(:defn, :"!@", s(:args), s(:nil)),
1790
+
1791
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1792
+ :tUBANG, "!@", EXPR_ARG, 0, 0,
1793
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1794
+ :kEND, "end", EXPR_END, 0, 0)
1795
+ end
1796
+
1797
+ def test_yylex_not_at_ivar
1798
+ assert_lex("!@ivar",
1799
+ s(:call, s(:ivar, :@ivar), :"!"),
1800
+
1801
+ :tBANG, "!", EXPR_BEG, 0, 0,
1802
+ :tIVAR, "@ivar", EXPR_END, 0, 0)
1803
+ end
1804
+
1805
+ def test_yylex_not_unary_method
1806
+ self.lex_state = EXPR_FNAME
1807
+
1808
+ assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
1683
1809
  end
1684
1810
 
1685
1811
  def test_yylex_nth_ref
1686
1812
  assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
1687
1813
  nil,
1688
- :tLBRACK, "[", :expr_beg,
1689
- :tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1690
- :tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1691
- :tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
1692
- :tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
1693
- :tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
1694
- :tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
1695
- :tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
1696
- :tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
1697
- :tNTH_REF, 9, :expr_end,
1698
- :tRBRACK, "]", :expr_endarg)
1814
+ :tLBRACK, "[", EXPR_PAR,
1815
+ :tNTH_REF, 1, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1816
+ :tNTH_REF, 2, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1817
+ :tNTH_REF, 3, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1818
+ :tNTH_REF, 4, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1819
+ :tNTH_REF, 5, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1820
+ :tNTH_REF, 6, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1821
+ :tNTH_REF, 7, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1822
+ :tNTH_REF, 8, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1823
+ :tNTH_REF, 9, EXPR_END,
1824
+ :tRBRACK, "]", EXPR_END)
1825
+ end
1826
+
1827
+ def test_yylex_number_times_ident_times_return_number
1828
+ assert_lex("1 * b * 3",
1829
+ s(:call,
1830
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1831
+ :*, s(:lit, 3)),
1832
+
1833
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1834
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1835
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1836
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1837
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1838
+
1839
+ assert_lex("1 * b *\n 3",
1840
+ s(:call,
1841
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1842
+ :*, s(:lit, 3)),
1843
+
1844
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1845
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1846
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1847
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1848
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1849
+ end
1850
+
1851
+ def test_yylex_numbers
1852
+ assert_lex3 "0b10", nil, :tINTEGER, 2, EXPR_NUM
1853
+ assert_lex3 "0B10", nil, :tINTEGER, 2, EXPR_NUM
1854
+
1855
+ assert_lex3 "0d10", nil, :tINTEGER, 10, EXPR_NUM
1856
+ assert_lex3 "0D10", nil, :tINTEGER, 10, EXPR_NUM
1857
+
1858
+ assert_lex3 "0x10", nil, :tINTEGER, 16, EXPR_NUM
1859
+ assert_lex3 "0X10", nil, :tINTEGER, 16, EXPR_NUM
1860
+
1861
+ assert_lex3 "0o10", nil, :tINTEGER, 8, EXPR_NUM
1862
+ assert_lex3 "0O10", nil, :tINTEGER, 8, EXPR_NUM
1863
+
1864
+ assert_lex3 "0o", nil, :tINTEGER, 0, EXPR_NUM
1865
+ assert_lex3 "0O", nil, :tINTEGER, 0, EXPR_NUM
1866
+
1867
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1868
+
1869
+ refute_lex "0x"
1870
+ refute_lex "0X"
1871
+ refute_lex "0b"
1872
+ refute_lex "0B"
1873
+ refute_lex "0d"
1874
+ refute_lex "0D"
1875
+
1876
+ refute_lex "08"
1877
+ refute_lex "09"
1878
+ refute_lex "0o8"
1879
+ refute_lex "0o9"
1880
+ refute_lex "0O8"
1881
+ refute_lex "0O9"
1882
+
1883
+ refute_lex "1_e1"
1884
+ refute_lex "1_.1"
1885
+ refute_lex "1__1"
1699
1886
  end
1700
1887
 
1701
1888
  def test_yylex_open_bracket
1702
- assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
1889
+ assert_lex3("(", nil, :tLPAREN, "(", EXPR_PAR)
1703
1890
  end
1704
1891
 
1705
1892
  def test_yylex_open_bracket_cmdarg
1706
- self.lex_state = :expr_cmdarg
1893
+ self.lex_state = EXPR_CMDARG
1707
1894
 
1708
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1895
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1709
1896
  end
1710
1897
 
1711
1898
  def test_yylex_open_bracket_exprarg__20
1712
1899
  setup_lexer_class RubyParser::V20
1713
- self.lex_state = :expr_arg
1900
+ self.lex_state = EXPR_ARG
1714
1901
 
1715
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1902
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1716
1903
  end
1717
1904
 
1718
1905
  def test_yylex_open_curly_bracket
1719
- assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
1906
+ assert_lex3("{", nil, :tLBRACE, "{", EXPR_PAR)
1720
1907
  end
1721
1908
 
1722
1909
  def test_yylex_open_curly_bracket_arg
1723
- self.lex_state = :expr_arg
1910
+ self.lex_state = EXPR_ARG
1724
1911
 
1725
1912
  assert_lex3("m { 3 }",
1726
1913
  nil,
1727
- :tIDENTIFIER, "m", :expr_cmdarg,
1728
- :tLCURLY, "{", :expr_beg,
1729
- :tINTEGER, 3, :expr_end,
1730
- :tRCURLY, "}", :expr_endarg)
1914
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1915
+ :tLCURLY, "{", EXPR_PAR,
1916
+ :tINTEGER, 3, EXPR_NUM,
1917
+ :tRCURLY, "}", EXPR_END)
1731
1918
  end
1732
1919
 
1733
1920
  def test_yylex_open_curly_bracket_block
1734
- self.lex_state = :expr_endarg # seen m(3)
1921
+ self.lex_state = EXPR_ENDARG # seen m(3)
1735
1922
 
1736
1923
  assert_lex3("{ 4 }",
1737
1924
  nil,
1738
- :tLBRACE_ARG, "{", :expr_beg,
1739
- :tINTEGER, 4, :expr_end,
1740
- :tRCURLY, "}", :expr_endarg)
1925
+ :tLBRACE_ARG, "{", EXPR_BEG,
1926
+ :tINTEGER, 4, EXPR_NUM,
1927
+ :tRCURLY, "}", EXPR_END)
1741
1928
  end
1742
1929
 
1743
1930
  def test_yylex_open_square_bracket_arg
1744
- self.lex_state = :expr_arg
1931
+ self.lex_state = EXPR_ARG
1745
1932
 
1746
1933
  assert_lex3("m [ 3 ]",
1747
1934
  nil,
1748
- :tIDENTIFIER, "m", :expr_cmdarg,
1749
- :tLBRACK, "[", :expr_beg,
1750
- :tINTEGER, 3, :expr_end,
1751
- :tRBRACK, "]", :expr_endarg)
1935
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1936
+ :tLBRACK, "[", EXPR_PAR,
1937
+ :tINTEGER, 3, EXPR_NUM,
1938
+ :tRBRACK, "]", EXPR_END)
1752
1939
  end
1753
1940
 
1754
1941
  def test_yylex_open_square_bracket_ary
1755
1942
  assert_lex3("[1, 2, 3]",
1756
1943
  nil,
1757
- :tLBRACK, "[", :expr_beg,
1758
- :tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1759
- :tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1760
- :tINTEGER, 3, :expr_end,
1761
- :tRBRACK, "]", :expr_endarg)
1944
+ :tLBRACK, "[", EXPR_PAR,
1945
+ :tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1946
+ :tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
1947
+ :tINTEGER, 3, EXPR_NUM,
1948
+ :tRBRACK, "]", EXPR_END)
1762
1949
  end
1763
1950
 
1764
1951
  def test_yylex_open_square_bracket_meth
1765
1952
  assert_lex3("m[3]",
1766
1953
  nil,
1767
- :tIDENTIFIER, "m", :expr_cmdarg,
1768
- :tLBRACK2, "[", :expr_beg,
1769
- :tINTEGER, 3, :expr_end,
1770
- :tRBRACK, "]", :expr_endarg)
1954
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1955
+ :tLBRACK2, "[", EXPR_PAR,
1956
+ :tINTEGER, 3, EXPR_NUM,
1957
+ :tRBRACK, "]", EXPR_END)
1771
1958
  end
1772
1959
 
1773
1960
  def test_yylex_or
1774
- assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
1961
+ assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
1775
1962
  end
1776
1963
 
1777
1964
  def test_yylex_or2
1778
- assert_lex3("||", nil, :tOROP, "||", :expr_beg)
1965
+ assert_lex3("||", nil, :tOROP, "||", EXPR_BEG)
1779
1966
  end
1780
1967
 
1781
1968
  def test_yylex_or2_equals
1782
- assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
1969
+ assert_lex3("||=", nil, :tOP_ASGN, "||", EXPR_BEG)
1783
1970
  end
1784
1971
 
1785
1972
  def test_yylex_or_equals
1786
- assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
1973
+ assert_lex3("|=", nil, :tOP_ASGN, "|", EXPR_BEG)
1787
1974
  end
1788
1975
 
1789
- def test_yylex_percent
1790
- assert_lex3("a % 2",
1791
- nil,
1792
- :tIDENTIFIER, "a", :expr_cmdarg,
1793
- :tPERCENT, "%", :expr_beg,
1794
- :tINTEGER, 2, :expr_end)
1795
- end
1976
+ def test_yylex_paren_string_interpolated_regexp
1977
+ setup_lexer('%( #{(/abcd/)} )',
1978
+ s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
1796
1979
 
1797
- def test_yylex_percent_equals
1798
- assert_lex3("a %= 2",
1799
- nil,
1800
- :tIDENTIFIER, "a", :expr_cmdarg,
1801
- :tOP_ASGN, "%", :expr_beg,
1802
- :tINTEGER, 2, :expr_end)
1803
- end
1980
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
1981
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1982
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1804
1983
 
1805
- def test_yylex_plus
1806
- assert_lex3("1 + 1", # TODO lex_state?
1807
- nil,
1808
- :tINTEGER, 1, :expr_end,
1809
- :tPLUS, "+", :expr_beg,
1810
- :tINTEGER, 1, :expr_end)
1811
- end
1984
+ emulate_string_interpolation do
1985
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
1986
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
1987
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
1988
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
1989
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
1990
+ end
1812
1991
 
1813
- def test_yylex_plus_equals
1814
- assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
1992
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1993
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1994
+
1995
+ refute_lexeme
1815
1996
  end
1816
1997
 
1817
- def test_yylex_plus_method
1818
- self.lex_state = :expr_fname
1998
+ def test_yylex_paren_string_parens_interpolated
1999
+ setup_lexer('%((#{b}#{d}))',
2000
+ s(:dstr,
2001
+ "(",
2002
+ s(:evstr, s(:call, nil, :b)),
2003
+ s(:evstr, s(:call, nil, :d)),
2004
+ s(:str, ")")))
1819
2005
 
1820
- assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
1821
- end
2006
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2007
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2008
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1822
2009
 
1823
- def test_yylex_plus_unary_method
1824
- self.lex_state = :expr_fname
2010
+ emulate_string_interpolation do
2011
+ assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2012
+ end
1825
2013
 
1826
- assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
1827
- end
2014
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1828
2015
 
1829
- def test_yylex_not_unary_method
1830
- self.lex_state = :expr_fname
2016
+ emulate_string_interpolation do
2017
+ assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
2018
+ end
2019
+
2020
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2021
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1831
2022
 
1832
- assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
2023
+ refute_lexeme
1833
2024
  end
1834
2025
 
1835
- def test_yylex_numbers
1836
- assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
1837
- assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
2026
+ def test_yylex_paren_string_parens_interpolated_regexp
2027
+ setup_lexer('%((#{(/abcd/)}))',
2028
+ s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
1838
2029
 
1839
- assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
1840
- assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
2030
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2031
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
1841
2032
 
1842
- assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
1843
- assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
2033
+ assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
1844
2034
 
1845
- assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
1846
- assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
2035
+ emulate_string_interpolation do
2036
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
2037
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
2038
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
2039
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
2040
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
2041
+ end
1847
2042
 
1848
- assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
1849
- assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
2043
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2044
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
1850
2045
 
1851
- assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
2046
+ refute_lexeme
2047
+ end
1852
2048
 
1853
- refute_lex "0x"
1854
- refute_lex "0X"
1855
- refute_lex "0b"
1856
- refute_lex "0B"
1857
- refute_lex "0d"
1858
- refute_lex "0D"
2049
+ def test_yylex_percent
2050
+ assert_lex3("a % 2",
2051
+ nil,
2052
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2053
+ :tPERCENT, "%", EXPR_BEG,
2054
+ :tINTEGER, 2, EXPR_NUM)
2055
+ end
1859
2056
 
1860
- refute_lex "08"
1861
- refute_lex "09"
1862
- refute_lex "0o8"
1863
- refute_lex "0o9"
1864
- refute_lex "0O8"
1865
- refute_lex "0O9"
2057
+ def test_yylex_percent_equals
2058
+ assert_lex3("a %= 2",
2059
+ nil,
2060
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2061
+ :tOP_ASGN, "%", EXPR_BEG,
2062
+ :tINTEGER, 2, EXPR_NUM)
2063
+ end
2064
+
2065
+ def test_yylex_plus
2066
+ assert_lex3("1 + 1", # TODO lex_state?
2067
+ nil,
2068
+ :tINTEGER, 1, EXPR_NUM,
2069
+ :tPLUS, "+", EXPR_BEG,
2070
+ :tINTEGER, 1, EXPR_NUM)
2071
+ end
2072
+
2073
+ def test_yylex_plus_equals
2074
+ assert_lex3("+=", nil, :tOP_ASGN, "+", EXPR_BEG)
2075
+ end
2076
+
2077
+ def test_yylex_plus_method
2078
+ self.lex_state = EXPR_FNAME
1866
2079
 
1867
- refute_lex "1_e1"
1868
- refute_lex "1_.1"
1869
- refute_lex "1__1"
2080
+ assert_lex3("+", nil, :tPLUS, "+", EXPR_ARG)
2081
+ end
2082
+
2083
+ def test_yylex_plus_unary_method
2084
+ self.lex_state = EXPR_FNAME
2085
+
2086
+ assert_lex3("+@", nil, :tUPLUS, "+@", EXPR_ARG)
1870
2087
  end
1871
2088
 
1872
2089
  def test_yylex_plus_unary_number
1873
- assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
2090
+ assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
1874
2091
  end
1875
2092
 
1876
2093
  def test_yylex_question_bad_eos
1877
2094
  refute_lex "?"
1878
2095
  end
1879
2096
 
2097
+ def test_yylex_question_eh_a__20
2098
+ setup_lexer_class RubyParser::V20
2099
+
2100
+ assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
2101
+ end
2102
+
2103
+ def test_yylex_question_eh_escape_M_escape_C__20
2104
+ setup_lexer_class RubyParser::V20
2105
+
2106
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
2107
+ end
2108
+
2109
+ def test_yylex_question_control_escape
2110
+ assert_lex3('?\C-\]', nil, :tSTRING, ?\C-\], EXPR_END)
2111
+ end
2112
+
1880
2113
  def test_yylex_question_ws
1881
- assert_lex3("? ", nil, :tEH, "?", :expr_value)
1882
- assert_lex3("?\n", nil, :tEH, "?", :expr_value)
1883
- assert_lex3("?\t", nil, :tEH, "?", :expr_value)
1884
- assert_lex3("?\v", nil, :tEH, "?", :expr_value)
1885
- assert_lex3("?\r", nil, :tEH, "?", :expr_value)
1886
- assert_lex3("?\f", nil, :tEH, "?", :expr_value)
2114
+ assert_lex3("? ", nil, :tEH, "?", EXPR_BEG)
2115
+ assert_lex3("?\n", nil, :tEH, "?", EXPR_BEG)
2116
+ assert_lex3("?\t", nil, :tEH, "?", EXPR_BEG)
2117
+ assert_lex3("?\v", nil, :tEH, "?", EXPR_BEG)
2118
+ assert_lex3("?\r", nil, :tEH, "?", EXPR_BEG)
2119
+ assert_lex3("?\f", nil, :tEH, "?", EXPR_BEG)
1887
2120
  end
1888
2121
 
1889
2122
  def test_yylex_question_ws_backslashed__20
1890
2123
  setup_lexer_class RubyParser::V20
1891
2124
 
1892
- assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
1893
- assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
1894
- assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
1895
- assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
1896
- assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
1897
- assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
2125
+ assert_lex3("?\\ ", nil, :tSTRING, " ", EXPR_END)
2126
+ assert_lex3("?\\n", nil, :tSTRING, "\n", EXPR_END)
2127
+ assert_lex3("?\\t", nil, :tSTRING, "\t", EXPR_END)
2128
+ assert_lex3("?\\v", nil, :tSTRING, "\v", EXPR_END)
2129
+ assert_lex3("?\\r", nil, :tSTRING, "\r", EXPR_END)
2130
+ assert_lex3("?\\f", nil, :tSTRING, "\f", EXPR_END)
1898
2131
  end
1899
2132
 
1900
2133
  def test_yylex_rbracket
1901
- assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
2134
+ assert_lex3("]", nil, :tRBRACK, "]", EXPR_END)
1902
2135
  end
1903
2136
 
1904
2137
  def test_yylex_rcurly
1905
- assert_lex("}", nil, :tRCURLY, "}", :expr_endarg, 0, 1) do
2138
+ assert_lex("}", nil, :tRCURLY, "}", EXPR_END, 0, 1) do
1906
2139
  lexer.brace_nest += 2
1907
2140
  end
1908
2141
  end
@@ -1910,18 +2143,18 @@ class TestRubyLexer < Minitest::Test
1910
2143
  def test_yylex_regexp
1911
2144
  assert_lex3("/regexp/",
1912
2145
  nil,
1913
- :tREGEXP_BEG, "/", :expr_beg,
1914
- :tSTRING_CONTENT, "regexp", :expr_beg,
1915
- :tREGEXP_END, "", :expr_end)
2146
+ :tREGEXP_BEG, "/", EXPR_BEG,
2147
+ :tSTRING_CONTENT, "regexp", EXPR_BEG,
2148
+ :tREGEXP_END, "", EXPR_LIT)
1916
2149
  end
1917
2150
 
1918
2151
  def test_yylex_regexp_ambiguous
1919
2152
  assert_lex3("method /regexp/",
1920
2153
  nil,
1921
- :tIDENTIFIER, "method", :expr_cmdarg,
1922
- :tREGEXP_BEG, "/", :expr_cmdarg,
1923
- :tSTRING_CONTENT, "regexp", :expr_cmdarg,
1924
- :tREGEXP_END, "", :expr_end)
2154
+ :tIDENTIFIER, "method", EXPR_CMDARG,
2155
+ :tREGEXP_BEG, "/", EXPR_CMDARG,
2156
+ :tSTRING_CONTENT, "regexp", EXPR_CMDARG,
2157
+ :tREGEXP_END, "", EXPR_LIT)
1925
2158
  end
1926
2159
 
1927
2160
  def test_yylex_regexp_bad
@@ -1933,25 +2166,25 @@ class TestRubyLexer < Minitest::Test
1933
2166
  def test_yylex_regexp_escape_C
1934
2167
  assert_lex3("/regex\\C-x/",
1935
2168
  nil,
1936
- :tREGEXP_BEG, "/", :expr_beg,
1937
- :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
1938
- :tREGEXP_END, "", :expr_end)
2169
+ :tREGEXP_BEG, "/", EXPR_BEG,
2170
+ :tSTRING_CONTENT, "regex\\C-x", EXPR_BEG,
2171
+ :tREGEXP_END, "", EXPR_LIT)
1939
2172
  end
1940
2173
 
1941
2174
  def test_yylex_regexp_escape_C_M
1942
2175
  assert_lex3("/regex\\C-\\M-x/",
1943
2176
  nil,
1944
- :tREGEXP_BEG, "/", :expr_beg,
1945
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1946
- :tREGEXP_END, "", :expr_end)
2177
+ :tREGEXP_BEG, "/", EXPR_BEG,
2178
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2179
+ :tREGEXP_END, "", EXPR_LIT)
1947
2180
  end
1948
2181
 
1949
2182
  def test_yylex_regexp_escape_C_M_craaaazy
1950
2183
  assert_lex3("/regex\\C-\\\n\\M-x/",
1951
2184
  nil,
1952
- :tREGEXP_BEG, "/", :expr_beg,
1953
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1954
- :tREGEXP_END, "", :expr_end)
2185
+ :tREGEXP_BEG, "/", EXPR_BEG,
2186
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2187
+ :tREGEXP_END, "", EXPR_LIT)
1955
2188
  end
1956
2189
 
1957
2190
  def test_yylex_regexp_escape_C_bad_dash
@@ -1977,17 +2210,17 @@ class TestRubyLexer < Minitest::Test
1977
2210
  def test_yylex_regexp_escape_M
1978
2211
  assert_lex3("/regex\\M-x/",
1979
2212
  nil,
1980
- :tREGEXP_BEG, "/", :expr_beg,
1981
- :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
1982
- :tREGEXP_END, "", :expr_end)
2213
+ :tREGEXP_BEG, "/", EXPR_BEG,
2214
+ :tSTRING_CONTENT, "regex\\M-x", EXPR_BEG,
2215
+ :tREGEXP_END, "", EXPR_LIT)
1983
2216
  end
1984
2217
 
1985
2218
  def test_yylex_regexp_escape_M_C
1986
2219
  assert_lex3("/regex\\M-\\C-x/",
1987
2220
  nil,
1988
- :tREGEXP_BEG, "/", :expr_beg,
1989
- :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
1990
- :tREGEXP_END, "", :expr_end)
2221
+ :tREGEXP_BEG, "/", EXPR_BEG,
2222
+ :tSTRING_CONTENT, "regex\\M-\\C-x", EXPR_BEG,
2223
+ :tREGEXP_END, "", EXPR_LIT)
1991
2224
  end
1992
2225
 
1993
2226
  def test_yylex_regexp_escape_M_bad_dash
@@ -2009,49 +2242,41 @@ class TestRubyLexer < Minitest::Test
2009
2242
  def test_yylex_regexp_escape_backslash_slash
2010
2243
  assert_lex3("/\\//",
2011
2244
  nil,
2012
- :tREGEXP_BEG, "/", :expr_beg,
2013
- :tSTRING_CONTENT, "\\/", :expr_beg,
2014
- :tREGEXP_END, "", :expr_end)
2245
+ :tREGEXP_BEG, "/", EXPR_BEG,
2246
+ :tSTRING_CONTENT, "\\/", EXPR_BEG,
2247
+ :tREGEXP_END, "", EXPR_LIT)
2015
2248
  end
2016
2249
 
2017
2250
  def test_yylex_regexp_escape_backslash_terminator
2018
2251
  assert_lex3("%r%blah\\%blah%",
2019
2252
  nil,
2020
- :tREGEXP_BEG, "%r\000", :expr_beg,
2021
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2022
- :tREGEXP_END, "", :expr_end)
2023
- end
2024
-
2025
- def test_yylex_regexp_escaped_delim
2026
- assert_lex3("%r!blah(?\\!blah)!",
2027
- nil,
2028
- :tREGEXP_BEG, "%r\000", :expr_beg,
2029
- :tSTRING_CONTENT, "blah(?!blah)", :expr_beg,
2030
- :tREGEXP_END, "", :expr_end)
2253
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2254
+ :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2255
+ :tREGEXP_END, "", EXPR_LIT)
2031
2256
  end
2032
2257
 
2033
2258
  def test_yylex_regexp_escape_backslash_terminator_meta1
2034
2259
  assert_lex3("%r{blah\\}blah}",
2035
2260
  nil,
2036
- :tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
2037
- :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
2038
- :tREGEXP_END, "", :expr_end)
2261
+ :tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
2262
+ :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2263
+ :tREGEXP_END, "", EXPR_LIT)
2039
2264
  end
2040
2265
 
2041
2266
  def test_yylex_regexp_escape_backslash_terminator_meta2
2042
2267
  assert_lex3("%r/blah\\/blah/",
2043
2268
  nil,
2044
- :tREGEXP_BEG, "%r\000", :expr_beg,
2045
- :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
2046
- :tREGEXP_END, "", :expr_end)
2269
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2270
+ :tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
2271
+ :tREGEXP_END, "", EXPR_LIT)
2047
2272
  end
2048
2273
 
2049
2274
  def test_yylex_regexp_escape_backslash_terminator_meta3
2050
2275
  assert_lex3("%r/blah\\%blah/",
2051
2276
  nil,
2052
- :tREGEXP_BEG, "%r\000", :expr_beg,
2053
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2054
- :tREGEXP_END, "", :expr_end)
2277
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2278
+ :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2279
+ :tREGEXP_END, "", EXPR_LIT)
2055
2280
  end
2056
2281
 
2057
2282
  def test_yylex_regexp_escape_bad_eos
@@ -2061,50 +2286,50 @@ class TestRubyLexer < Minitest::Test
2061
2286
  def test_yylex_regexp_escape_bs
2062
2287
  assert_lex3("/regex\\\\regex/",
2063
2288
  nil,
2064
- :tREGEXP_BEG, "/", :expr_beg,
2065
- :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
2066
- :tREGEXP_END, "", :expr_end)
2289
+ :tREGEXP_BEG, "/", EXPR_BEG,
2290
+ :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2291
+ :tREGEXP_END, "", EXPR_LIT)
2067
2292
  end
2068
2293
 
2069
2294
  def test_yylex_regexp_escape_c
2070
2295
  assert_lex3("/regex\\cxxx/",
2071
2296
  nil,
2072
- :tREGEXP_BEG, "/", :expr_beg,
2073
- :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
2074
- :tREGEXP_END, "", :expr_end)
2297
+ :tREGEXP_BEG, "/", EXPR_BEG,
2298
+ :tSTRING_CONTENT, "regex\\cxxx", EXPR_BEG,
2299
+ :tREGEXP_END, "", EXPR_LIT)
2075
2300
  end
2076
2301
 
2077
2302
  def test_yylex_regexp_escape_c_backslash
2078
2303
  assert_lex3("/regex\\c\\n/",
2079
2304
  nil,
2080
- :tREGEXP_BEG, "/", :expr_beg,
2081
- :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
2082
- :tREGEXP_END, "", :expr_end)
2305
+ :tREGEXP_BEG, "/", EXPR_BEG,
2306
+ :tSTRING_CONTENT, "regex\\c\\n", EXPR_BEG,
2307
+ :tREGEXP_END, "", EXPR_LIT)
2083
2308
  end
2084
2309
 
2085
2310
  def test_yylex_regexp_escape_chars
2086
2311
  assert_lex3("/re\\tge\\nxp/",
2087
2312
  nil,
2088
- :tREGEXP_BEG, "/", :expr_beg,
2089
- :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
2090
- :tREGEXP_END, "", :expr_end)
2313
+ :tREGEXP_BEG, "/", EXPR_BEG,
2314
+ :tSTRING_CONTENT, "re\\tge\\nxp", EXPR_BEG,
2315
+ :tREGEXP_END, "", EXPR_LIT)
2091
2316
  end
2092
2317
 
2093
2318
  def test_yylex_regexp_escape_double_backslash
2094
2319
  regexp = '/[\\/\\\\]$/'
2095
2320
  assert_lex3(regexp.dup,
2096
2321
  nil,
2097
- :tREGEXP_BEG, "/", :expr_beg,
2098
- :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
2099
- :tREGEXP_END, "", :expr_end)
2322
+ :tREGEXP_BEG, "/", EXPR_BEG,
2323
+ :tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
2324
+ :tREGEXP_END, "", EXPR_LIT)
2100
2325
  end
2101
2326
 
2102
2327
  def test_yylex_regexp_escape_hex
2103
2328
  assert_lex3("/regex\\x61xp/",
2104
2329
  nil,
2105
- :tREGEXP_BEG, "/", :expr_beg,
2106
- :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
2107
- :tREGEXP_END, "", :expr_end)
2330
+ :tREGEXP_BEG, "/", EXPR_BEG,
2331
+ :tSTRING_CONTENT, "regex\\x61xp", EXPR_BEG,
2332
+ :tREGEXP_END, "", EXPR_LIT)
2108
2333
  end
2109
2334
 
2110
2335
  def test_yylex_regexp_escape_hex_bad
@@ -2114,412 +2339,427 @@ class TestRubyLexer < Minitest::Test
2114
2339
  def test_yylex_regexp_escape_hex_one
2115
2340
  assert_lex3("/^[\\xd\\xa]{2}/on",
2116
2341
  nil,
2117
- :tREGEXP_BEG, "/", :expr_beg,
2118
- :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
2119
- :tREGEXP_END, "on", :expr_end)
2342
+ :tREGEXP_BEG, "/", EXPR_BEG,
2343
+ :tSTRING_CONTENT, "^[\\xd\\xa]{2}", EXPR_BEG,
2344
+ :tREGEXP_END, "on", EXPR_LIT)
2120
2345
  end
2121
2346
 
2122
2347
  def test_yylex_regexp_escape_oct1
2123
2348
  assert_lex3("/regex\\0xp/",
2124
2349
  nil,
2125
- :tREGEXP_BEG, "/", :expr_beg,
2126
- :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
2127
- :tREGEXP_END, "", :expr_end)
2350
+ :tREGEXP_BEG, "/", EXPR_BEG,
2351
+ :tSTRING_CONTENT, "regex\\0xp", EXPR_BEG,
2352
+ :tREGEXP_END, "", EXPR_LIT)
2128
2353
  end
2129
2354
 
2130
2355
  def test_yylex_regexp_escape_oct2
2131
2356
  assert_lex3("/regex\\07xp/",
2132
2357
  nil,
2133
- :tREGEXP_BEG, "/", :expr_beg,
2134
- :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
2135
- :tREGEXP_END, "", :expr_end)
2358
+ :tREGEXP_BEG, "/", EXPR_BEG,
2359
+ :tSTRING_CONTENT, "regex\\07xp", EXPR_BEG,
2360
+ :tREGEXP_END, "", EXPR_LIT)
2136
2361
  end
2137
2362
 
2138
2363
  def test_yylex_regexp_escape_oct3
2139
2364
  assert_lex3("/regex\\10142/",
2140
2365
  nil,
2141
- :tREGEXP_BEG, "/", :expr_beg,
2142
- :tSTRING_CONTENT, "regex\\10142", :expr_beg,
2143
- :tREGEXP_END, "", :expr_end)
2366
+ :tREGEXP_BEG, "/", EXPR_BEG,
2367
+ :tSTRING_CONTENT, "regex\\10142", EXPR_BEG,
2368
+ :tREGEXP_END, "", EXPR_LIT)
2144
2369
  end
2145
2370
 
2146
2371
  def test_yylex_regexp_escape_return
2147
2372
  assert_lex3("/regex\\\nregex/",
2148
2373
  nil,
2149
- :tREGEXP_BEG, "/", :expr_beg,
2150
- :tSTRING_CONTENT, "regexregex", :expr_beg,
2151
- :tREGEXP_END, "", :expr_end)
2374
+ :tREGEXP_BEG, "/", EXPR_BEG,
2375
+ :tSTRING_CONTENT, "regexregex", EXPR_BEG,
2376
+ :tREGEXP_END, "", EXPR_LIT)
2377
+ end
2378
+
2379
+ def test_yylex_regexp_escaped_delim
2380
+ assert_lex3("%r!blah(?\\!blah)!",
2381
+ nil,
2382
+ :tREGEXP_BEG, "%r\000", EXPR_BEG,
2383
+ :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2384
+ :tREGEXP_END, "", EXPR_LIT)
2152
2385
  end
2153
2386
 
2154
2387
  def test_yylex_regexp_nm
2155
2388
  assert_lex3("/.*/nm",
2156
2389
  nil,
2157
- :tREGEXP_BEG, "/", :expr_beg,
2158
- :tSTRING_CONTENT, ".*", :expr_beg,
2159
- :tREGEXP_END, "nm", :expr_end)
2390
+ :tREGEXP_BEG, "/", EXPR_BEG,
2391
+ :tSTRING_CONTENT, ".*", EXPR_BEG,
2392
+ :tREGEXP_END, "nm", EXPR_LIT)
2393
+ end
2394
+
2395
+ def test_yylex_required_kwarg_no_value_22
2396
+ setup_lexer_class RubyParser::V22
2397
+
2398
+ assert_lex3("def foo a:, b:\nend",
2399
+ nil,
2400
+ :kDEF, "def", EXPR_FNAME,
2401
+ :tIDENTIFIER, "foo", EXPR_ENDFN,
2402
+ :tLABEL, "a", EXPR_LAB,
2403
+ :tCOMMA, ",", EXPR_PAR,
2404
+ :tLABEL, "b", EXPR_LAB,
2405
+ :kEND, "end", EXPR_END)
2160
2406
  end
2161
2407
 
2162
2408
  def test_yylex_rparen
2163
- assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
2409
+ assert_lex3(")", nil, :tRPAREN, ")", EXPR_ENDFN)
2164
2410
  end
2165
2411
 
2166
2412
  def test_yylex_rshft
2167
2413
  assert_lex3("a >> 2",
2168
2414
  nil,
2169
- :tIDENTIFIER, "a", :expr_cmdarg,
2170
- :tRSHFT, ">>", :expr_beg,
2171
- :tINTEGER, 2, :expr_end)
2415
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2416
+ :tRSHFT, ">>", EXPR_BEG,
2417
+ :tINTEGER, 2, EXPR_NUM)
2172
2418
  end
2173
2419
 
2174
2420
  def test_yylex_rshft_equals
2175
2421
  assert_lex3("a >>= 2",
2176
2422
  nil,
2177
- :tIDENTIFIER, "a", :expr_cmdarg,
2178
- :tOP_ASGN, ">>", :expr_beg,
2179
- :tINTEGER, 2, :expr_end)
2423
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2424
+ :tOP_ASGN, ">>", EXPR_BEG,
2425
+ :tINTEGER, 2, EXPR_NUM)
2180
2426
  end
2181
2427
 
2182
2428
  def test_yylex_star
2183
2429
  assert_lex3("a * ",
2184
2430
  nil,
2185
- :tIDENTIFIER, "a", :expr_cmdarg,
2186
- :tSTAR2, "*", :expr_beg)
2431
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2432
+ :tSTAR2, "*", EXPR_BEG)
2187
2433
  end
2188
2434
 
2189
2435
  def test_yylex_star2
2190
2436
  assert_lex3("a ** ",
2191
2437
  nil,
2192
- :tIDENTIFIER, "a", :expr_cmdarg,
2193
- :tPOW, "**", :expr_beg)
2438
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2439
+ :tPOW, "**", EXPR_BEG)
2194
2440
  end
2195
2441
 
2196
2442
  def test_yylex_star2_equals
2197
2443
  assert_lex3("a **= ",
2198
2444
  nil,
2199
- :tIDENTIFIER, "a", :expr_cmdarg,
2200
- :tOP_ASGN, "**", :expr_beg)
2445
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2446
+ :tOP_ASGN, "**", EXPR_BEG)
2201
2447
  end
2202
2448
 
2203
2449
  def test_yylex_star_arg
2204
- self.lex_state = :expr_arg
2450
+ self.lex_state = EXPR_ARG
2205
2451
 
2206
2452
  assert_lex3(" *a",
2207
2453
  nil,
2208
- :tSTAR, "*", :expr_beg,
2209
- :tIDENTIFIER, "a", :expr_arg)
2454
+ :tSTAR, "*", EXPR_BEG,
2455
+ :tIDENTIFIER, "a", EXPR_ARG)
2210
2456
  end
2211
2457
 
2212
2458
  def test_yylex_star_arg_beg
2213
- self.lex_state = :expr_beg
2459
+ self.lex_state = EXPR_BEG
2214
2460
 
2215
2461
  assert_lex3("*a",
2216
2462
  nil,
2217
- :tSTAR, "*", :expr_beg,
2218
- :tIDENTIFIER, "a", :expr_arg)
2463
+ :tSTAR, "*", EXPR_BEG,
2464
+ :tIDENTIFIER, "a", EXPR_ARG)
2219
2465
  end
2220
2466
 
2221
2467
  def test_yylex_star_arg_beg_fname
2222
- self.lex_state = :expr_fname
2468
+ self.lex_state = EXPR_FNAME
2223
2469
 
2224
2470
  assert_lex3("*a",
2225
2471
  nil,
2226
- :tSTAR2, "*", :expr_arg,
2227
- :tIDENTIFIER, "a", :expr_arg)
2472
+ :tSTAR2, "*", EXPR_ARG,
2473
+ :tIDENTIFIER, "a", EXPR_ARG)
2228
2474
  end
2229
2475
 
2230
2476
  def test_yylex_star_arg_beg_fname2
2231
- self.lex_state = :expr_fname
2477
+ self.lex_state = EXPR_FNAME
2232
2478
 
2233
2479
  assert_lex3("*a",
2234
2480
  nil,
2235
- :tSTAR2, "*", :expr_arg,
2236
- :tIDENTIFIER, "a", :expr_arg)
2481
+ :tSTAR2, "*", EXPR_ARG,
2482
+ :tIDENTIFIER, "a", EXPR_ARG)
2237
2483
  end
2238
2484
 
2239
2485
  def test_yylex_star_equals
2240
2486
  assert_lex3("a *= ",
2241
2487
  nil,
2242
- :tIDENTIFIER, "a", :expr_cmdarg,
2243
- :tOP_ASGN, "*", :expr_beg)
2488
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2489
+ :tOP_ASGN, "*", EXPR_BEG)
2244
2490
  end
2245
2491
 
2246
2492
  def test_yylex_string_bad_eos
2247
- refute_lex('%', :tSTRING_BEG, '%')
2493
+ refute_lex("%", :tSTRING_BEG, "%")
2248
2494
  end
2249
2495
 
2250
2496
  def test_yylex_string_bad_eos_quote
2251
- refute_lex('%{nest', :tSTRING_BEG, '%}')
2497
+ refute_lex("%{nest", :tSTRING_BEG, "%}")
2252
2498
  end
2253
2499
 
2254
2500
  def test_yylex_string_double
2255
- assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
2501
+ assert_lex3("\"string\"", nil, :tSTRING, "string", EXPR_END)
2256
2502
  end
2257
2503
 
2258
2504
  def test_yylex_string_double_escape_C
2259
- assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
2505
+ assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", EXPR_END)
2260
2506
  end
2261
2507
 
2262
2508
  def test_yylex_string_double_escape_C_backslash
2263
2509
  assert_lex3("\"\\C-\\\\\"",
2264
2510
  nil,
2265
- :tSTRING_BEG, "\"", :expr_beg,
2266
- :tSTRING_CONTENT, "\034", :expr_beg,
2267
- :tSTRING_END, "\"", :expr_end)
2511
+ :tSTRING_BEG, "\"", EXPR_BEG,
2512
+ :tSTRING_CONTENT, "\034", EXPR_BEG,
2513
+ :tSTRING_END, "\"", EXPR_LIT)
2268
2514
  end
2269
2515
 
2270
2516
  def test_yylex_string_double_escape_C_escape
2271
2517
  assert_lex3("\"\\C-\\M-a\"",
2272
2518
  nil,
2273
- :tSTRING_BEG, "\"", :expr_beg,
2274
- :tSTRING_CONTENT, "\201", :expr_beg,
2275
- :tSTRING_END, "\"", :expr_end)
2519
+ :tSTRING_BEG, "\"", EXPR_BEG,
2520
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2521
+ :tSTRING_END, "\"", EXPR_LIT)
2276
2522
  end
2277
2523
 
2278
2524
  def test_yylex_string_double_escape_C_question
2279
- assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
2280
- end
2281
-
2282
- def test_yylex_string_utf8_simple
2283
- chr = [0x3024].pack("U")
2284
-
2285
- assert_lex3('"\u{3024}"',
2286
- s(:str, chr),
2287
- :tSTRING, chr, :expr_end)
2288
- end
2289
-
2290
- def test_yylex_string_utf8_complex
2291
- chr = [0x3024].pack("U")
2292
-
2293
- assert_lex3('"#@a\u{3024}"',
2294
- s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2295
- :tSTRING_BEG, '"', :expr_beg,
2296
- :tSTRING_DVAR, nil, :expr_beg,
2297
- :tSTRING_CONTENT, "@a"+chr, :expr_beg,
2298
- :tSTRING_END, '"', :expr_end)
2525
+ assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", EXPR_END)
2299
2526
  end
2300
2527
 
2301
2528
  def test_yylex_string_double_escape_M
2302
2529
  chr = "\341"
2303
- chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
2304
-
2305
- assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
2306
- end
2307
2530
 
2308
- def test_why_does_ruby_hate_me?
2309
- assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2310
- nil,
2311
- :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
2531
+ assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, EXPR_END)
2312
2532
  end
2313
2533
 
2314
2534
  def test_yylex_string_double_escape_M_backslash
2315
2535
  assert_lex3("\"\\M-\\\\\"",
2316
2536
  nil,
2317
- :tSTRING_BEG, "\"", :expr_beg,
2318
- :tSTRING_CONTENT, "\334", :expr_beg,
2319
- :tSTRING_END, "\"", :expr_end)
2537
+ :tSTRING_BEG, "\"", EXPR_BEG,
2538
+ :tSTRING_CONTENT, "\334", EXPR_BEG,
2539
+ :tSTRING_END, "\"", EXPR_LIT)
2320
2540
  end
2321
2541
 
2322
2542
  def test_yylex_string_double_escape_M_escape
2323
2543
  assert_lex3("\"\\M-\\C-a\"",
2324
2544
  nil,
2325
- :tSTRING_BEG, "\"", :expr_beg,
2326
- :tSTRING_CONTENT, "\201", :expr_beg,
2327
- :tSTRING_END, "\"", :expr_end)
2545
+ :tSTRING_BEG, "\"", EXPR_BEG,
2546
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2547
+ :tSTRING_END, "\"", EXPR_LIT)
2328
2548
  end
2329
2549
 
2330
2550
  def test_yylex_string_double_escape_bs1
2331
- assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
2551
+ assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", EXPR_END)
2332
2552
  end
2333
2553
 
2334
2554
  def test_yylex_string_double_escape_bs2
2335
- assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
2555
+ assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", EXPR_END)
2336
2556
  end
2337
2557
 
2338
2558
  def test_yylex_string_double_escape_c
2339
- assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
2559
+ assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", EXPR_END)
2340
2560
  end
2341
2561
 
2342
2562
  def test_yylex_string_double_escape_c_backslash
2343
- assert_lex3("\"\\c\\\"",
2344
- nil,
2345
- :tSTRING_BEG, "\"", :expr_beg,
2346
- :tSTRING_CONTENT, "\034", :expr_beg,
2347
- :tSTRING_END, "\"", :expr_end)
2563
+ refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
2348
2564
  end
2349
2565
 
2350
2566
  def test_yylex_string_double_escape_c_escape
2351
2567
  assert_lex3("\"\\c\\M-a\"",
2352
2568
  nil,
2353
- :tSTRING_BEG, "\"", :expr_beg,
2354
- :tSTRING_CONTENT, "\201", :expr_beg,
2355
- :tSTRING_END, "\"", :expr_end)
2569
+ :tSTRING_BEG, "\"", EXPR_BEG,
2570
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2571
+ :tSTRING_END, "\"", EXPR_LIT)
2356
2572
  end
2357
2573
 
2358
2574
  def test_yylex_string_double_escape_c_question
2359
- assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
2575
+ assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", EXPR_END)
2360
2576
  end
2361
2577
 
2362
2578
  def test_yylex_string_double_escape_chars
2363
- assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
2579
+ assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", EXPR_END)
2364
2580
  end
2365
2581
 
2366
2582
  def test_yylex_string_double_escape_hex
2367
- assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
2583
+ assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", EXPR_END)
2368
2584
  end
2369
2585
 
2370
2586
  def test_yylex_string_double_escape_octal
2371
- assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
2587
+ assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", EXPR_END)
2372
2588
  end
2373
2589
 
2374
2590
  def test_yylex_string_double_escape_octal_fucked
2375
- assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
2591
+ assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", EXPR_END)
2376
2592
  end
2377
2593
 
2378
2594
  def test_yylex_string_double_interp
2379
2595
  assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
2380
2596
  nil,
2381
- :tSTRING_BEG, "\"", :expr_beg,
2382
- :tSTRING_CONTENT, "blah #x a ", :expr_beg,
2383
- :tSTRING_DVAR, nil, :expr_beg,
2384
- :tSTRING_CONTENT, "@a b ", :expr_beg,
2385
- :tSTRING_DVAR, nil, :expr_beg,
2386
- :tSTRING_CONTENT, "$b c ", :expr_beg,
2387
- :tSTRING_DBEG, nil, :expr_beg,
2388
- :tSTRING_CONTENT, "3} # ", :expr_beg,
2389
- :tSTRING_END, "\"", :expr_end)
2390
- end
2391
-
2392
- def test_yylex_string_double_pound_dollar_bad
2393
- assert_lex3('"#$%"', nil,
2394
-
2395
- :tSTRING_BEG, "\"", :expr_beg,
2396
- :tSTRING_CONTENT, '#$%', :expr_beg,
2397
- :tSTRING_END, "\"", :expr_end)
2597
+ :tSTRING_BEG, "\"", EXPR_BEG,
2598
+ :tSTRING_CONTENT, "blah #x a ", EXPR_BEG,
2599
+ :tSTRING_DVAR, nil, EXPR_BEG,
2600
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG,
2601
+ :tSTRING_DVAR, nil, EXPR_BEG,
2602
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2603
+ :tSTRING_DBEG, nil, EXPR_BEG,
2604
+ :tSTRING_CONTENT, "3} # ", EXPR_BEG,
2605
+ :tSTRING_END, "\"", EXPR_LIT)
2398
2606
  end
2399
2607
 
2400
2608
  def test_yylex_string_double_nested_curlies
2401
2609
  assert_lex3("%{nest{one{two}one}nest}",
2402
2610
  nil,
2403
- :tSTRING_BEG, "%}", :expr_beg,
2404
- :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
2405
- :tSTRING_END, "}", :expr_end)
2611
+ :tSTRING_BEG, "%}", EXPR_BEG,
2612
+ :tSTRING_CONTENT, "nest{one{two}one}nest", EXPR_BEG,
2613
+ :tSTRING_END, "}", EXPR_LIT)
2406
2614
  end
2407
2615
 
2408
2616
  def test_yylex_string_double_no_interp
2409
- assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
2410
- assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
2617
+ assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", EXPR_END)
2618
+ assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", EXPR_END)
2411
2619
  end
2412
2620
 
2413
- def test_yylex_string_escape_x_single
2414
- assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
2621
+ def test_yylex_string_double_pound_dollar_bad
2622
+ assert_lex3('"#$%"', nil,
2623
+
2624
+ :tSTRING_BEG, "\"", EXPR_BEG,
2625
+ :tSTRING_CONTENT, "#\$%", EXPR_BEG,
2626
+ :tSTRING_END, "\"", EXPR_LIT)
2415
2627
  end
2416
2628
 
2417
- def test_yylex_string_pct_i
2418
- assert_lex3("%i[s1 s2\ns3]",
2419
- nil,
2420
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2421
- :tSTRING_CONTENT, "s1", :expr_beg,
2422
- :tSPACE, nil, :expr_beg,
2423
- :tSTRING_CONTENT, "s2", :expr_beg,
2424
- :tSPACE, nil, :expr_beg,
2425
- :tSTRING_CONTENT, "s3", :expr_beg,
2426
- :tSPACE, nil, :expr_beg,
2427
- :tSTRING_END, nil, :expr_end)
2629
+ def test_yylex_string_escape_x_single
2630
+ assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", EXPR_END)
2428
2631
  end
2429
2632
 
2430
2633
  def test_yylex_string_pct_I
2431
2634
  assert_lex3("%I[s1 s2\ns3]",
2432
2635
  nil,
2433
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2434
- :tSTRING_CONTENT, "s1", :expr_beg,
2435
- :tSPACE, nil, :expr_beg,
2436
- :tSTRING_CONTENT, "s2", :expr_beg,
2437
- :tSPACE, nil, :expr_beg,
2438
- :tSTRING_CONTENT, "s3", :expr_beg,
2439
- :tSPACE, nil, :expr_beg,
2440
- :tSTRING_END, nil, :expr_end)
2636
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2637
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2638
+ :tSPACE, nil, EXPR_BEG,
2639
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2640
+ :tSPACE, nil, EXPR_BEG,
2641
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2642
+ :tSPACE, nil, EXPR_BEG,
2643
+ :tSTRING_END, nil, EXPR_LIT)
2441
2644
  end
2442
2645
 
2443
- def test_yylex_string_pct_i_extra_space
2444
- assert_lex3("%i[ s1 s2\ns3 ]",
2646
+ def test_yylex_string_pct_I_extra_space
2647
+ assert_lex3("%I[ s1 s2\ns3 ]",
2445
2648
  nil,
2446
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2447
- :tSTRING_CONTENT, "s1", :expr_beg,
2448
- :tSPACE, nil, :expr_beg,
2449
- :tSTRING_CONTENT, "s2", :expr_beg,
2450
- :tSPACE, nil, :expr_beg,
2451
- :tSTRING_CONTENT, "s3", :expr_beg,
2452
- :tSPACE, nil, :expr_beg,
2453
- :tSTRING_END, nil, :expr_end)
2649
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2650
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2651
+ :tSPACE, nil, EXPR_BEG,
2652
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2653
+ :tSPACE, nil, EXPR_BEG,
2654
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2655
+ :tSPACE, nil, EXPR_BEG,
2656
+ :tSTRING_END, nil, EXPR_LIT)
2454
2657
  end
2455
2658
 
2456
- def test_yylex_string_pct_I_extra_space
2457
- assert_lex3("%I[ s1 s2\ns3 ]",
2659
+ def test_yylex_string_pct_Q
2660
+ assert_lex3("%Q[s1 s2]",
2458
2661
  nil,
2459
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2460
- :tSTRING_CONTENT, "s1", :expr_beg,
2461
- :tSPACE, nil, :expr_beg,
2462
- :tSTRING_CONTENT, "s2", :expr_beg,
2463
- :tSPACE, nil, :expr_beg,
2464
- :tSTRING_CONTENT, "s3", :expr_beg,
2465
- :tSPACE, nil, :expr_beg,
2466
- :tSTRING_END, nil, :expr_end)
2662
+ :tSTRING_BEG, "%Q[", EXPR_BEG,
2663
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2664
+ :tSTRING_END, "]", EXPR_LIT)
2467
2665
  end
2468
2666
 
2469
- def test_yylex_string_pct_q
2470
- assert_lex3("%q[s1 s2]",
2667
+ def test_yylex_string_pct_Q_null_wtf?
2668
+ assert_lex3("%Q\0s1 s2\0",
2471
2669
  nil,
2472
- :tSTRING_BEG, "%q[", :expr_beg,
2473
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2474
- :tSTRING_END, "]", :expr_end)
2670
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2671
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2672
+ :tSTRING_END, "\0", EXPR_LIT)
2475
2673
  end
2476
2674
 
2477
- def test_yylex_string_pct_Q
2478
- assert_lex3("%Q[s1 s2]",
2675
+ def test_yylex_string_pct_Q_bang
2676
+ assert_lex3("%Q!s1 s2!",
2479
2677
  nil,
2480
- :tSTRING_BEG, "%Q[", :expr_beg,
2481
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2482
- :tSTRING_END, "]", :expr_end)
2678
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2679
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2680
+ :tSTRING_END, "!", EXPR_LIT)
2483
2681
  end
2484
2682
 
2485
2683
  def test_yylex_string_pct_W
2486
2684
  assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2487
2685
  nil,
2488
- :tWORDS_BEG, "%W[", :expr_beg,
2489
- :tSTRING_CONTENT, "s1", :expr_beg,
2490
- :tSPACE, nil, :expr_beg,
2491
- :tSTRING_CONTENT, "s2", :expr_beg,
2492
- :tSPACE, nil, :expr_beg,
2493
- :tSTRING_CONTENT, "s3", :expr_beg,
2494
- :tSPACE, nil, :expr_beg,
2495
- :tSTRING_END, nil, :expr_end)
2686
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2687
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2688
+ :tSPACE, nil, EXPR_BEG,
2689
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2690
+ :tSPACE, nil, EXPR_BEG,
2691
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2692
+ :tSPACE, nil, EXPR_BEG,
2693
+ :tSTRING_END, nil, EXPR_LIT)
2496
2694
  end
2497
2695
 
2498
2696
  def test_yylex_string_pct_W_bs_nl
2499
2697
  assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2500
2698
  nil,
2501
- :tWORDS_BEG, "%W[", :expr_beg,
2502
- :tSTRING_CONTENT, "s1", :expr_beg,
2503
- :tSPACE, nil, :expr_beg,
2504
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2505
- :tSPACE, nil, :expr_beg,
2506
- :tSTRING_END, nil, :expr_end)
2699
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2700
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2701
+ :tSPACE, nil, EXPR_BEG,
2702
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2703
+ :tSPACE, nil, EXPR_BEG,
2704
+ :tSTRING_END, nil, EXPR_LIT)
2507
2705
  end
2508
2706
 
2509
2707
  def test_yylex_string_pct_angle
2510
2708
  assert_lex3("%<blah>",
2511
2709
  nil,
2512
- :tSTRING_BEG, "%>", :expr_beg,
2513
- :tSTRING_CONTENT, "blah", :expr_beg,
2514
- :tSTRING_END, ">", :expr_end)
2710
+ :tSTRING_BEG, "%>", EXPR_BEG,
2711
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2712
+ :tSTRING_END, ">", EXPR_LIT)
2713
+ end
2714
+
2715
+ def test_yylex_string_pct_i
2716
+ assert_lex3("%i[s1 s2\ns3]",
2717
+ nil,
2718
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2719
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2720
+ :tSPACE, nil, EXPR_BEG,
2721
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2722
+ :tSPACE, nil, EXPR_BEG,
2723
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2724
+ :tSPACE, nil, EXPR_BEG,
2725
+ :tSTRING_END, nil, EXPR_LIT)
2726
+ end
2727
+
2728
+ def test_yylex_string_pct_i_extra_space
2729
+ assert_lex3("%i[ s1 s2\ns3 ]",
2730
+ nil,
2731
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2732
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2733
+ :tSPACE, nil, EXPR_BEG,
2734
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2735
+ :tSPACE, nil, EXPR_BEG,
2736
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2737
+ :tSPACE, nil, EXPR_BEG,
2738
+ :tSTRING_END, nil, EXPR_LIT)
2515
2739
  end
2516
2740
 
2517
2741
  def test_yylex_string_pct_other
2518
2742
  assert_lex3("%%blah%",
2519
2743
  nil,
2520
- :tSTRING_BEG, "%%", :expr_beg,
2521
- :tSTRING_CONTENT, "blah", :expr_beg,
2522
- :tSTRING_END, "%", :expr_end)
2744
+ :tSTRING_BEG, "%%", EXPR_BEG,
2745
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2746
+ :tSTRING_END, "%", EXPR_LIT)
2747
+ end
2748
+
2749
+ def test_yylex_string_pct_q
2750
+ assert_lex3("%q[s1 s2]",
2751
+ nil,
2752
+ :tSTRING_BEG, "%q[", EXPR_BEG,
2753
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2754
+ :tSTRING_END, "]", EXPR_LIT)
2755
+ end
2756
+
2757
+ def test_yylex_string_pct_s
2758
+ assert_lex3("%s[s1 s2]",
2759
+ nil,
2760
+ :tSYMBEG, "%s[", EXPR_FNAME, # TODO: :tSYM_BEG ?
2761
+ :tSTRING_CONTENT, "s1 s2", EXPR_FNAME, # man... I don't like this
2762
+ :tSTRING_END, "]", EXPR_LIT)
2523
2763
  end
2524
2764
 
2525
2765
  def test_yylex_string_pct_w
@@ -2534,130 +2774,222 @@ class TestRubyLexer < Minitest::Test
2534
2774
  def test_yylex_string_pct_w_bs_nl
2535
2775
  assert_lex3("%w[s1 \\\ns2]",
2536
2776
  nil,
2537
- :tQWORDS_BEG, "%w[", :expr_beg,
2538
- :tSTRING_CONTENT, "s1", :expr_beg,
2539
- :tSPACE, nil, :expr_beg,
2540
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2541
- :tSPACE, nil, :expr_beg,
2542
- :tSTRING_END, nil, :expr_end)
2777
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2778
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2779
+ :tSPACE, nil, EXPR_BEG,
2780
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2781
+ :tSPACE, nil, EXPR_BEG,
2782
+ :tSTRING_END, nil, EXPR_LIT)
2543
2783
  end
2544
2784
 
2545
2785
  def test_yylex_string_pct_w_bs_sp
2546
2786
  assert_lex3("%w[s\\ 1 s\\ 2]",
2547
2787
  nil,
2548
- :tQWORDS_BEG, "%w[", :expr_beg,
2549
- :tSTRING_CONTENT, "s 1", :expr_beg,
2550
- :tSPACE, nil, :expr_beg,
2551
- :tSTRING_CONTENT, "s 2", :expr_beg,
2552
- :tSPACE, nil, :expr_beg,
2553
- :tSTRING_END, nil, :expr_end)
2788
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2789
+ :tSTRING_CONTENT, "s 1", EXPR_BEG,
2790
+ :tSPACE, nil, EXPR_BEG,
2791
+ :tSTRING_CONTENT, "s 2", EXPR_BEG,
2792
+ :tSPACE, nil, EXPR_BEG,
2793
+ :tSTRING_END, nil, EXPR_LIT)
2554
2794
  end
2555
2795
 
2556
2796
  def test_yylex_string_single
2557
- assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
2797
+ assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
2558
2798
  end
2559
2799
 
2560
2800
  def test_yylex_string_single_escape_chars
2561
- assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
2801
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
2562
2802
  end
2563
2803
 
2564
- def test_yylex_string_single_nl
2565
- assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
2804
+ def test_yylex_string_single_escape_quote_and_backslash
2805
+ assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz",
2806
+ EXPR_LIT)
2566
2807
  end
2567
2808
 
2568
2809
  def test_yylex_string_single_escaped_quote
2569
- assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", :expr_end)
2810
+ assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
2570
2811
  end
2571
2812
 
2572
- def test_yylex_symbol
2573
- assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
2813
+ def test_yylex_string_single_nl
2814
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
2574
2815
  end
2575
2816
 
2576
- def test_yylex_symbol_zero_byte
2577
- assert_lex(":\"symbol\0\"", nil,
2578
- :tSYMBOL, "symbol\0", :expr_end)
2817
+ def test_yylex_string_utf8_complex
2818
+ chr = [0x3024].pack("U")
2819
+
2820
+ assert_lex3('"#@a\u{3024}"',
2821
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2822
+ :tSTRING_BEG, '"', EXPR_BEG,
2823
+ :tSTRING_DVAR, nil, EXPR_BEG,
2824
+ :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2825
+ :tSTRING_END, '"', EXPR_LIT)
2826
+ end
2827
+
2828
+ def test_yylex_string_utf8_complex_missing_hex
2829
+ chr = [0x302].pack("U")
2830
+ str = "#{chr}zzz"
2831
+
2832
+ refute_lex('"#@a\u302zzz"',
2833
+ :tSTRING_BEG, '"',
2834
+ :tSTRING_DVAR, nil,
2835
+ :tSTRING_CONTENT, "@a"+str,
2836
+ :tSTRING_END, '"')
2837
+
2838
+ chr = [0x30].pack("U")
2839
+ str = "#{chr}zzz"
2840
+
2841
+ refute_lex('"#@a\u30zzz"',
2842
+ :tSTRING_BEG, '"',
2843
+ :tSTRING_DVAR, nil,
2844
+ :tSTRING_CONTENT, "@a"+str,
2845
+ :tSTRING_END, '"')
2846
+
2847
+ chr = [0x3].pack("U")
2848
+ str = "#{chr}zzz"
2849
+
2850
+ refute_lex('"#@a\u3zzz"',
2851
+ :tSTRING_BEG, '"',
2852
+ :tSTRING_DVAR, nil,
2853
+ :tSTRING_CONTENT, "@a"+str,
2854
+ :tSTRING_END, '"')
2855
+ end
2856
+
2857
+ def test_yylex_string_utf8_bad_encoding_with_escapes
2858
+ str = "\"\\xBADπ\""
2859
+ exp = "\xBADπ".b
2860
+
2861
+ assert_lex(str,
2862
+ s(:str, exp),
2863
+ :tSTRING, exp, EXPR_END)
2864
+ end
2865
+
2866
+ def test_yylex_string_utf8_complex_trailing_hex
2867
+ chr = [0x3024].pack("U")
2868
+ str = "#{chr}abz"
2869
+
2870
+ assert_lex3('"#@a\u3024abz"',
2871
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2872
+ :tSTRING_BEG, '"', EXPR_BEG,
2873
+ :tSTRING_DVAR, nil, EXPR_BEG,
2874
+ :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2875
+ :tSTRING_END, '"', EXPR_LIT)
2876
+ end
2877
+
2878
+ def test_yylex_string_utf8_missing_hex
2879
+ refute_lex('"\u3zzz"')
2880
+ refute_lex('"\u30zzz"')
2881
+ refute_lex('"\u302zzz"')
2882
+ end
2883
+
2884
+ def test_yylex_string_utf8_simple
2885
+ chr = [0x3024].pack("U")
2886
+
2887
+ assert_lex3('"\u{3024}"',
2888
+ s(:str, chr),
2889
+ :tSTRING, chr, EXPR_END)
2890
+ end
2891
+
2892
+ def test_yylex_string_utf8_trailing_hex
2893
+ chr = [0x3024].pack("U")
2894
+ str = "#{chr}abz"
2895
+
2896
+ assert_lex3('"\u3024abz"',
2897
+ s(:str, str),
2898
+ :tSTRING, str, EXPR_END)
2899
+ end
2900
+
2901
+ def test_yylex_sym_quoted
2902
+ assert_lex(":'a'",
2903
+ s(:lit, :a),
2904
+
2905
+ :tSYMBOL, "a", EXPR_LIT, 0, 0)
2906
+ end
2907
+
2908
+ def test_yylex_symbol
2909
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_LIT)
2579
2910
  end
2580
2911
 
2581
2912
  def test_yylex_symbol_double
2582
2913
  assert_lex3(":\"symbol\"",
2583
2914
  nil,
2584
- :tSYMBOL, "symbol", :expr_end)
2915
+ :tSYMBOL, "symbol", EXPR_LIT)
2585
2916
  end
2586
2917
 
2587
2918
  def test_yylex_symbol_double_interp
2588
2919
  assert_lex3(':"symbol#{1+1}"',
2589
2920
  nil,
2590
- :tSYMBEG, ":", :expr_fname,
2591
- :tSTRING_CONTENT, "symbol", :expr_fname,
2592
- :tSTRING_DBEG, nil, :expr_fname,
2593
- :tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
2594
- :tSTRING_END, "\"", :expr_end)
2921
+ :tSYMBEG, ":", EXPR_FNAME,
2922
+ :tSTRING_CONTENT, "symbol", EXPR_FNAME,
2923
+ :tSTRING_DBEG, nil, EXPR_FNAME,
2924
+ :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
2925
+ :tSTRING_END, "\"", EXPR_LIT)
2595
2926
  end
2596
2927
 
2597
2928
  def test_yylex_symbol_single
2598
2929
  assert_lex3(":'symbol'",
2599
2930
  nil,
2600
- :tSYMBOL, "symbol", :expr_end)
2931
+ :tSYMBOL, "symbol", EXPR_LIT)
2601
2932
  end
2602
2933
 
2603
- def test_yylex_symbol_single_noninterp
2604
- assert_lex3(':\'symbol#{1+1}\'',
2934
+ def test_yylex_symbol_single_escape_chars
2935
+ assert_lex3(":'s\\tri\\ng'",
2605
2936
  nil,
2606
- :tSYMBOL, 'symbol#{1+1}', :expr_end)
2937
+ :tSYMBOL, "s\\tri\\ng", EXPR_LIT)
2607
2938
  end
2608
2939
 
2609
- def test_yylex_symbol_single_escape_chars
2610
- assert_lex3(":'s\\tri\\ng'",
2940
+ def test_yylex_symbol_single_noninterp
2941
+ assert_lex3(':\'symbol#{1+1}\'',
2611
2942
  nil,
2612
- :tSYMBOL, "s\\tri\\ng", :expr_end)
2943
+ :tSYMBOL, 'symbol#{1+1}', EXPR_LIT)
2613
2944
  end
2614
2945
 
2615
- def test_yylex_string_single_escape_quote_and_backslash
2616
- assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz", :expr_end)
2946
+ def test_yylex_symbol_zero_byte
2947
+ assert_lex(":\"symbol\0\"", nil,
2948
+ :tSYMBOL, "symbol\0", EXPR_LIT)
2617
2949
  end
2618
2950
 
2619
2951
  def test_yylex_ternary1
2620
2952
  assert_lex3("a ? b : c",
2621
2953
  nil,
2622
- :tIDENTIFIER, "a", :expr_cmdarg,
2623
- :tEH, "?", :expr_value,
2624
- :tIDENTIFIER, "b", :expr_arg,
2625
- :tCOLON, ":", :expr_beg,
2626
- :tIDENTIFIER, "c", :expr_arg)
2954
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2955
+ :tEH, "?", EXPR_BEG,
2956
+ :tIDENTIFIER, "b", EXPR_ARG,
2957
+ :tCOLON, ":", EXPR_BEG,
2958
+ :tIDENTIFIER, "c", EXPR_ARG)
2627
2959
 
2628
2960
  assert_lex3("a ?bb : c", # GAH! MATZ!!!
2629
2961
  nil,
2630
- :tIDENTIFIER, "a", :expr_cmdarg,
2631
- :tEH, "?", :expr_beg,
2632
- :tIDENTIFIER, "bb", :expr_arg,
2633
- :tCOLON, ":", :expr_beg,
2634
- :tIDENTIFIER, "c", :expr_arg)
2962
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2963
+ :tEH, "?", EXPR_BEG,
2964
+ :tIDENTIFIER, "bb", EXPR_ARG,
2965
+ :tCOLON, ":", EXPR_BEG,
2966
+ :tIDENTIFIER, "c", EXPR_ARG)
2635
2967
 
2636
2968
  assert_lex3("42 ?",
2637
2969
  nil,
2638
- :tINTEGER, 42, :expr_end,
2639
- :tEH, "?", :expr_value)
2970
+ :tINTEGER, 42, EXPR_NUM,
2971
+ :tEH, "?", EXPR_BEG)
2640
2972
  end
2641
2973
 
2642
2974
  def test_yylex_tilde
2643
- assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
2975
+ assert_lex3("~", nil, :tTILDE, "~", EXPR_BEG)
2644
2976
  end
2645
2977
 
2646
2978
  def test_yylex_tilde_unary
2647
- self.lex_state = :expr_fname
2979
+ self.lex_state = EXPR_FNAME
2648
2980
 
2649
- assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
2981
+ assert_lex3("~@", nil, :tTILDE, "~", EXPR_ARG)
2650
2982
  end
2651
2983
 
2652
2984
  def test_yylex_uminus
2653
2985
  assert_lex3("-blah",
2654
2986
  nil,
2655
- :tUMINUS, "-", :expr_beg,
2656
- :tIDENTIFIER, "blah", :expr_arg)
2987
+ :tUMINUS, "-", EXPR_BEG,
2988
+ :tIDENTIFIER, "blah", EXPR_ARG)
2657
2989
  end
2658
2990
 
2659
2991
  def test_yylex_underscore
2660
- assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
2992
+ assert_lex3("_var", nil, :tIDENTIFIER, "_var", EXPR_CMDARG)
2661
2993
  end
2662
2994
 
2663
2995
  def test_yylex_underscore_end
@@ -2669,229 +3001,45 @@ class TestRubyLexer < Minitest::Test
2669
3001
  def test_yylex_uplus
2670
3002
  assert_lex3("+blah",
2671
3003
  nil,
2672
- :tUPLUS, "+", :expr_beg,
2673
- :tIDENTIFIER, "blah", :expr_arg)
3004
+ :tUPLUS, "+", EXPR_BEG,
3005
+ :tIDENTIFIER, "blah", EXPR_ARG)
2674
3006
  end
2675
3007
 
2676
3008
  def test_zbug_float_in_decl
2677
3009
  assert_lex3("def initialize(u = 0.0, s = 0.0",
2678
3010
  nil,
2679
- :kDEF, "def", :expr_fname,
2680
- :tIDENTIFIER, "initialize", :expr_endfn,
2681
- :tLPAREN2, "(", :expr_beg,
2682
- :tIDENTIFIER, "u", :expr_arg,
2683
- :tEQL, "=", :expr_beg,
2684
- :tFLOAT, 0.0, :expr_end,
2685
- :tCOMMA, ",", :expr_beg,
2686
- :tIDENTIFIER, "s", :expr_arg,
2687
- :tEQL, "=", :expr_beg,
2688
- :tFLOAT, 0.0, :expr_end)
3011
+ :kDEF, "def", EXPR_FNAME,
3012
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3013
+ :tLPAREN2, "(", EXPR_PAR,
3014
+ :tIDENTIFIER, "u", EXPR_ARG,
3015
+ :tEQL, "=", EXPR_BEG,
3016
+ :tFLOAT, 0.0, EXPR_NUM,
3017
+ :tCOMMA, ",", EXPR_PAR,
3018
+ :tIDENTIFIER, "s", EXPR_ARG,
3019
+ :tEQL, "=", EXPR_BEG,
3020
+ :tFLOAT, 0.0, EXPR_NUM)
2689
3021
  end
2690
3022
 
2691
3023
  def test_zbug_id_equals
2692
3024
  assert_lex3("a = 0.0",
2693
3025
  nil,
2694
- :tIDENTIFIER, "a", :expr_cmdarg,
2695
- :tEQL, "=", :expr_beg,
2696
- :tFLOAT, 0.0, :expr_end)
3026
+ :tIDENTIFIER, "a", EXPR_CMDARG,
3027
+ :tEQL, "=", EXPR_BEG,
3028
+ :tFLOAT, 0.0, EXPR_NUM)
2697
3029
  end
2698
3030
 
2699
3031
  def test_zbug_no_spaces_in_decl
2700
3032
  assert_lex3("def initialize(u=0.0,s=0.0",
2701
3033
  nil,
2702
- :kDEF, "def", :expr_fname,
2703
- :tIDENTIFIER, "initialize", :expr_endfn,
2704
- :tLPAREN2, "(", :expr_beg,
2705
- :tIDENTIFIER, "u", :expr_arg,
2706
- :tEQL, "=", :expr_beg,
2707
- :tFLOAT, 0.0, :expr_end,
2708
- :tCOMMA, ",", :expr_beg,
2709
- :tIDENTIFIER, "s", :expr_arg,
2710
- :tEQL, "=", :expr_beg,
2711
- :tFLOAT, 0.0, :expr_end)
2712
- end
2713
-
2714
- def test_pct_w_backslashes
2715
- ["\t", "\n", "\r", "\v", "\f"].each do |char|
2716
- next if !RubyLexer::HAS_ENC and char == "\v"
2717
-
2718
- assert_lex("%w[foo#{char}bar]",
2719
- s(:array, s(:str, "foo"), s(:str, "bar")),
2720
-
2721
- :tQWORDS_BEG, "%w[", :expr_beg, 0, 0,
2722
- :tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
2723
- :tSPACE, nil, :expr_beg, 0, 0,
2724
- :tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
2725
- :tSPACE, nil, :expr_beg, 0, 0,
2726
- :tSTRING_END, nil, :expr_end, 0, 0)
2727
- end
2728
- end
2729
-
2730
- def test_yylex_sym_quoted
2731
- assert_lex(":'a'",
2732
- s(:lit, :a),
2733
-
2734
- :tSYMBOL, "a", :expr_end, 0, 0)
2735
- end
2736
-
2737
- def test_yylex_hash_colon
2738
- assert_lex("{a:1}",
2739
- s(:hash, s(:lit, :a), s(:lit, 1)),
2740
-
2741
- :tLBRACE, "{", :expr_beg, 0, 1,
2742
- :tLABEL, "a", :expr_labeled, 0, 1,
2743
- :tINTEGER, 1, :expr_end, 0, 1,
2744
- :tRCURLY, "}", :expr_endarg, 0, 0)
2745
- end
2746
-
2747
- def test_yylex_hash_colon_quoted_22
2748
- setup_lexer_class RubyParser::V22
2749
-
2750
- assert_lex("{'a':1}",
2751
- s(:hash, s(:lit, :a), s(:lit, 1)),
2752
-
2753
- :tLBRACE, "{", :expr_beg, 0, 1,
2754
- :tLABEL, "a", :expr_labeled, 0, 1,
2755
- :tINTEGER, 1, :expr_end, 0, 1,
2756
- :tRCURLY, "}", :expr_endarg, 0, 0)
2757
- end
2758
-
2759
- def test_yylex_hash_colon_quoted_symbol_22
2760
- setup_lexer_class RubyParser::V22
2761
-
2762
- assert_lex("{'abc': :b}",
2763
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2764
-
2765
- :tLBRACE, "{", :expr_beg, 0, 1,
2766
- :tLABEL, "abc", :expr_labeled, 0, 1,
2767
- :tSYMBOL, "b", :expr_end, 0, 1,
2768
- :tRCURLY, "}", :expr_endarg, 0, 0)
2769
- end
2770
-
2771
- def test_yylex_hash_colon_double_quoted_symbol_22
2772
- setup_lexer_class RubyParser::V22
2773
-
2774
- assert_lex('{"abc": :b}',
2775
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2776
-
2777
- :tLBRACE, "{", :expr_beg, 0, 1,
2778
- :tLABEL, "abc", :expr_labeled, 0, 1,
2779
- :tSYMBOL, "b", :expr_end, 0, 1,
2780
- :tRCURLY, "}", :expr_endarg, 0, 0)
2781
- end
2782
-
2783
- def test_yylex_required_kwarg_no_value_22
2784
- setup_lexer_class RubyParser::V22
2785
-
2786
- assert_lex3("def foo a:, b:\nend",
2787
- nil,
2788
- :kDEF, "def", :expr_fname,
2789
- :tIDENTIFIER, "foo", :expr_endfn,
2790
- :tLABEL, "a", :expr_labeled,
2791
- :tCOMMA, ",", :expr_beg,
2792
- :tLABEL, "b", :expr_labeled,
2793
- :kEND, "end", :expr_end)
2794
- end
2795
-
2796
- def test_yylex_hash_colon_double_quoted_with_escapes
2797
- setup_lexer_class RubyParser::V22
2798
-
2799
- assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
2800
- nil,
2801
-
2802
- :tLBRACE, "{", :expr_beg,
2803
- :tLABEL, "s\tr\i\ng\\foo'bar", :expr_labeled,
2804
- :tINTEGER, 1, :expr_end,
2805
- :tRCURLY, "}", :expr_endarg)
2806
- end
2807
-
2808
- def test_yylex_hash_colon_quoted_with_escapes
2809
- setup_lexer_class RubyParser::V22
2810
-
2811
- assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
2812
- nil,
2813
-
2814
- :tLBRACE, "{", :expr_beg,
2815
- :tLABEL, "s\\tr\\i\\ng\\foo'bar", :expr_labeled,
2816
- :tINTEGER, 1, :expr_end,
2817
- :tRCURLY, "}", :expr_endarg)
2818
- end
2819
-
2820
- def test_ruby21_rational_literal
2821
- setup_lexer_class RubyParser::V21
2822
-
2823
- assert_lex3("10r", nil, :tRATIONAL, Rational(10), :expr_end)
2824
- assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), :expr_end)
2825
- assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), :expr_end)
2826
- assert_lex3("0or", nil, :tRATIONAL, Rational(0), :expr_end)
2827
- assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), :expr_end)
2828
- assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), :expr_end)
2829
- assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), :expr_end)
2830
- assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), :expr_end)
2831
- assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), :expr_end)
2832
- assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), :expr_end)
2833
-
2834
- assert_lex3("-10r", nil,
2835
- :tUMINUS_NUM, "-", :expr_beg,
2836
- :tRATIONAL, Rational(10), :expr_end)
2837
- end
2838
-
2839
- def test_ruby21_imaginary_literal
2840
- setup_lexer_class RubyParser::V21
2841
-
2842
- assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), :expr_end)
2843
- assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), :expr_end)
2844
- assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), :expr_end)
2845
- assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), :expr_end)
2846
- assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), :expr_end)
2847
- assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
2848
- assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
2849
- assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
2850
- assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
2851
- assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
2852
-
2853
- assert_lex3("-10i", nil,
2854
- :tUMINUS_NUM, "-", :expr_beg,
2855
- :tIMAGINARY, Complex(0, 10), :expr_end)
2856
- end
2857
-
2858
- def test_ruby21_rational_imaginary_literal
2859
- setup_lexer_class RubyParser::V21
2860
-
2861
- assert_lex3("1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
2862
- assert_lex3("0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
2863
- assert_lex3("0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
2864
- assert_lex3("0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
2865
- assert_lex3("0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
2866
- assert_lex3("1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
2867
- assert_lex3("15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
2868
- assert_lex3("15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
2869
- assert_lex3("1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
2870
- assert_lex3("1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
2871
-
2872
- assert_lex3("-10ri", nil,
2873
- :tUMINUS_NUM, "-", :expr_beg,
2874
- :tIMAGINARY, Complex(0, Rational(10)), :expr_end)
2875
- end
2876
-
2877
- def test_ruby21_imaginary_literal_with_succeeding_keyword
2878
- skip "Currently does not tokenize correctly"
2879
-
2880
- setup_lexer_class RubyParser::V21
2881
-
2882
- assert_lex3("1if", nil,
2883
- :tINTEGER, 1, :expr_end,
2884
- :kIF_MOD, "if", :expr_beg)
2885
- assert_lex3("1rif", nil,
2886
- :tRATIONAL, Rational(1), :expr_end,
2887
- :kIF_MOD, "if", :expr_beg)
2888
- assert_lex3("1.0if", nil,
2889
- :tFLOAT, 1.0, :expr_end,
2890
- :kIF_MOD, "if", :expr_beg)
2891
- assert_lex3("1.0rif", nil,
2892
- :tRATIONAL, Rational("1.0"), :expr_end,
2893
- :kIF_MOD, "if", :expr_beg)
2894
-
2895
- flunk
3034
+ :kDEF, "def", EXPR_FNAME,
3035
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3036
+ :tLPAREN2, "(", EXPR_PAR,
3037
+ :tIDENTIFIER, "u", EXPR_ARG,
3038
+ :tEQL, "=", EXPR_BEG,
3039
+ :tFLOAT, 0.0, EXPR_NUM,
3040
+ :tCOMMA, ",", EXPR_PAR,
3041
+ :tIDENTIFIER, "s", EXPR_ARG,
3042
+ :tEQL, "=", EXPR_BEG,
3043
+ :tFLOAT, 0.0, EXPR_NUM)
2896
3044
  end
2897
3045
  end