ruby_parser 3.12.0 → 3.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.autotest +18 -29
  4. data/History.rdoc +283 -0
  5. data/Manifest.txt +12 -4
  6. data/README.rdoc +4 -3
  7. data/Rakefile +189 -51
  8. data/bin/ruby_parse +3 -1
  9. data/bin/ruby_parse_extract_error +19 -36
  10. data/compare/normalize.rb +76 -4
  11. data/debugging.md +190 -0
  12. data/gauntlet.md +106 -0
  13. data/lib/rp_extensions.rb +14 -42
  14. data/lib/rp_stringscanner.rb +20 -51
  15. data/lib/ruby20_parser.rb +4659 -4218
  16. data/lib/ruby20_parser.y +953 -602
  17. data/lib/ruby21_parser.rb +4723 -4308
  18. data/lib/ruby21_parser.y +956 -605
  19. data/lib/ruby22_parser.rb +4762 -4337
  20. data/lib/ruby22_parser.y +960 -612
  21. data/lib/ruby23_parser.rb +4761 -4342
  22. data/lib/ruby23_parser.y +961 -613
  23. data/lib/ruby24_parser.rb +4791 -4341
  24. data/lib/ruby24_parser.y +968 -612
  25. data/lib/ruby25_parser.rb +4791 -4341
  26. data/lib/ruby25_parser.y +968 -612
  27. data/lib/ruby26_parser.rb +7287 -0
  28. data/lib/ruby26_parser.y +2749 -0
  29. data/lib/ruby27_parser.rb +8517 -0
  30. data/lib/ruby27_parser.y +3346 -0
  31. data/lib/ruby30_parser.rb +8751 -0
  32. data/lib/ruby30_parser.y +3472 -0
  33. data/lib/ruby3_parser.yy +3476 -0
  34. data/lib/ruby_lexer.rb +611 -826
  35. data/lib/ruby_lexer.rex +48 -40
  36. data/lib/ruby_lexer.rex.rb +122 -46
  37. data/lib/ruby_lexer_strings.rb +638 -0
  38. data/lib/ruby_parser.rb +38 -34
  39. data/lib/ruby_parser.yy +1710 -704
  40. data/lib/ruby_parser_extras.rb +987 -553
  41. data/test/test_ruby_lexer.rb +1718 -1539
  42. data/test/test_ruby_parser.rb +3957 -2164
  43. data/test/test_ruby_parser_extras.rb +39 -4
  44. data/tools/munge.rb +250 -0
  45. data/tools/ripper.rb +44 -0
  46. data.tar.gz.sig +0 -0
  47. metadata +68 -47
  48. metadata.gz.sig +0 -0
  49. data/lib/ruby18_parser.rb +0 -5793
  50. data/lib/ruby18_parser.y +0 -1908
  51. data/lib/ruby19_parser.rb +0 -6185
  52. data/lib/ruby19_parser.y +0 -2116
data/test/test_ruby_lexer.rb CHANGED
@@ -1,29 +1,24 @@
1
- # encoding: US-ASCII
2
-
3
1
  require "minitest/autorun"
4
2
  require "ruby_lexer"
5
3
  require "ruby_parser"
6
4
 
7
5
  class TestRubyLexer < Minitest::Test
6
+ include RubyLexer::State::Values
7
+
8
8
  attr_accessor :processor, :lex, :parser_class, :lex_state
9
9
 
10
- alias :lexer :lex # lets me copy/paste code from parser
11
- alias :lexer= :lex=
10
+ alias lexer lex # lets me copy/paste code from parser
11
+ alias lexer= lex=
12
12
 
13
13
  def setup
14
- self.lex_state = :expr_beg
14
+ self.lex_state = EXPR_BEG
15
15
  setup_lexer_class RubyParser.latest.class
16
16
  end
17
17
 
18
18
  def setup_lexer input, exp_sexp = nil
19
19
  setup_new_parser
20
20
  lex.ss = RPStringScanner.new(input)
21
- lex.lex_state = self.lex_state
22
- end
23
-
24
- def setup_new_parser
25
- self.processor = parser_class.new
26
- self.lex = processor.lexer
21
+ lex.lex_state = lex_state
27
22
  end
28
23
 
29
24
  def setup_lexer_class parser_class
@@ -32,11 +27,16 @@ class TestRubyLexer < Minitest::Test
32
27
  setup_lexer "blah blah"
33
28
  end
34
29
 
35
- def assert_lex input, exp_sexp, *args, &b
30
+ def setup_new_parser
31
+ self.processor = parser_class.new
32
+ self.lex = processor.lexer
33
+ end
34
+
35
+ def assert_lex input, exp_sexp, *args
36
36
  setup_lexer input
37
37
  assert_parse input, exp_sexp if exp_sexp
38
38
 
39
- b.call if b
39
+ yield if block_given?
40
40
 
41
41
  args.each_slice(5) do |token, value, state, paren, brace|
42
42
  assert_next_lexeme token, value, state, paren, brace
@@ -46,57 +46,56 @@ class TestRubyLexer < Minitest::Test
46
46
  end
47
47
 
48
48
  def assert_lex3 input, exp_sexp, *args, &block
49
+ # TODO: refute_nil exp_sexp, "Get off your lazy butt and write one"
50
+
49
51
  args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
50
52
 
51
53
  assert_lex(input, exp_sexp, *args, &block)
52
54
  end
53
55
 
54
- def ruby18
55
- RubyParser::V18 === lexer.parser
56
- end
57
-
58
- def refute_lex input, *args # TODO: re-sort
59
- args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
56
+ def refute_lex3 input, *args # TODO: re-sort
57
+ args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
60
58
 
61
59
  assert_raises RubyParser::SyntaxError do
62
60
  assert_lex(input, nil, *args)
63
61
  end
64
62
  end
65
63
 
66
- def assert_lex_fname name, type, end_state = :expr_arg # TODO: swap name/type
64
+ def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
67
65
  assert_lex3("def #{name} ",
68
66
  nil,
69
67
 
70
- :kDEF, "def", :expr_fname,
68
+ :kDEF, "def", EXPR_FNAME,
71
69
  type, name, end_state)
72
70
  end
73
71
 
74
72
  def assert_next_lexeme token=nil, value=nil, state=nil, paren=nil, brace=nil
75
73
  adv = @lex.next_token
76
74
 
77
- assert adv, "no more tokens"
75
+ assert adv, "no more tokens, expecting: %p %p %p %p %p" % [token, value, state, paren, brace]
78
76
 
79
77
  act_token, act_value = adv
80
78
 
81
79
  msg = message {
82
- act = [act_token, act_value, @lex.lex_state,
83
- @lex.paren_nest, @lex.brace_nest]
80
+ act = [act_token, act_value, @lex.lex_state, @lex.paren_nest, @lex.brace_nest]
84
81
  exp = [token, value, state, paren, brace]
85
82
  "#{exp.inspect} vs #{act.inspect}"
86
83
  }
87
84
 
88
85
  act_value = act_value.first if Array === act_value
89
86
 
90
- assert_equal token, act_token, msg
87
+ assert_equal token, act_token, msg
91
88
  case value
92
89
  when Float then
93
90
  assert_in_epsilon value, act_value, 0.001, msg
94
91
  when NilClass then
95
92
  assert_nil act_value, msg
93
+ when String then
94
+ assert_equal value, act_value.b.force_encoding(value.encoding), msg
96
95
  else
97
- assert_equal value, act_value, msg
96
+ assert_equal value, act_value, msg
98
97
  end
99
- assert_equal state, @lex.lex_state, msg if state
98
+ assert_match state, @lex.lex_state, msg if state
100
99
  assert_equal paren, @lex.paren_nest, msg if paren
101
100
  assert_equal brace, @lex.brace_nest, msg if brace
102
101
  end
@@ -106,21 +105,36 @@ class TestRubyLexer < Minitest::Test
106
105
  end
107
106
 
108
107
  def assert_read_escape expected, input
109
- @lex.ss.string = input
110
- assert_equal expected, @lex.read_escape, input
108
+ setup_lexer input
109
+ enc = expected.encoding
110
+ assert_equal expected, lex.read_escape.b.force_encoding(enc), input
111
111
  end
112
112
 
113
113
  def assert_read_escape_bad input # TODO: rename refute_read_escape
114
- @lex.ss.string = input
114
+ setup_lexer input
115
+ assert_raises RubyParser::SyntaxError do
116
+ lex.read_escape
117
+ end
118
+ end
119
+
120
+ def refute_lex input, *args # TODO: re-sort
121
+ args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
122
+
123
+ assert_raises RubyParser::SyntaxError do
124
+ assert_lex(input, nil, *args)
125
+ end
126
+ end
127
+
128
+ def refute_lex5 input, *args
115
129
  assert_raises RubyParser::SyntaxError do
116
- @lex.read_escape
130
+ assert_lex(input, *args)
117
131
  end
118
132
  end
119
133
 
120
134
  def refute_lexeme
121
135
  x = y = @lex.next_token
122
136
 
123
- refute x, "not empty: #{y.inspect}"
137
+ refute x, "not empty: #{y.inspect}: #{@lex.rest.inspect}"
124
138
  end
125
139
 
126
140
  ## Utility Methods:
@@ -136,15 +150,15 @@ class TestRubyLexer < Minitest::Test
136
150
  lexer.cmdarg.push false
137
151
 
138
152
  lexer.lex_strterm = nil
139
- lexer.lex_state = :expr_beg
153
+ lexer.lex_state = EXPR_BEG
140
154
 
141
155
  yield
142
156
 
143
- lexer.lex_state = :expr_endarg
144
- assert_next_lexeme :tRCURLY, "}", :expr_endarg, 0
157
+ lexer.lex_state = EXPR_ENDARG
158
+ assert_next_lexeme :tSTRING_DEND, "}", EXPR_END|EXPR_ENDARG, 0
145
159
 
146
160
  lexer.lex_strterm = lex_strterm
147
- lexer.lex_state = :expr_beg
161
+ lexer.lex_state = EXPR_BEG
148
162
  lexer.string_nest = string_nest
149
163
  lexer.brace_nest = brace_nest
150
164
 
@@ -155,14 +169,25 @@ class TestRubyLexer < Minitest::Test
155
169
  ## Tests:
156
170
 
157
171
  def test_next_token
158
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
159
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
172
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
173
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
160
174
  assert_nil @lex.next_token
161
175
  end
162
176
 
163
- def test_unicode_ident
164
- s = "@\u1088\u1077\u1093\u1072"
165
- assert_lex3(s.dup, nil, :tIVAR, s.dup, :expr_end)
177
+ def test_pct_w_backslashes
178
+ ["\t", "\n", "\r", "\v", "\f"].each do |char|
179
+ next if !RubyLexer::HAS_ENC and char == "\v"
180
+
181
+ assert_lex("%w[foo#{char}bar]",
182
+ s(:array, s(:str, "foo"), s(:str, "bar")),
183
+
184
+ :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
185
+ :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
186
+ :tSPACE, " ", EXPR_BEG, 0, 0,
187
+ :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
188
+ :tSPACE, "]", EXPR_BEG, 0, 0,
189
+ :tSTRING_END, "]", EXPR_LIT, 0, 0)
190
+ end
166
191
  end
167
192
 
168
193
  def test_read_escape
@@ -218,12 +243,96 @@ class TestRubyLexer < Minitest::Test
218
243
  assert_read_escape "\230", 'M-\cx'
219
244
  end
220
245
 
246
+ def test_ruby21_imaginary_literal
247
+ setup_lexer_class RubyParser::V21
248
+
249
+ assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), EXPR_NUM)
250
+ assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), EXPR_NUM)
251
+ assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), EXPR_NUM)
252
+ assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), EXPR_NUM)
253
+ assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), EXPR_NUM)
254
+ assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), EXPR_NUM)
255
+ assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), EXPR_NUM)
256
+ assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), EXPR_NUM)
257
+ assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), EXPR_NUM)
258
+ assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), EXPR_NUM)
259
+
260
+ c010 = Complex(0, 10)
261
+ assert_lex3("-10i", nil,
262
+ :tUMINUS_NUM, "-", EXPR_BEG,
263
+ :tIMAGINARY, c010, EXPR_NUM)
264
+ end
265
+
266
+ def test_ruby21_imaginary_literal_with_succeeding_keyword
267
+ setup_lexer_class RubyParser::V21
268
+
269
+ # 2/4 scenarios are syntax errors on all tested versions so I
270
+ # deleted them.
271
+
272
+ assert_lex3("1if", nil,
273
+ :tINTEGER, 1, EXPR_NUM,
274
+ :kIF_MOD, "if", EXPR_PAR)
275
+ assert_lex3("1.0if", nil,
276
+ :tFLOAT, 1.0, EXPR_NUM,
277
+ :kIF_MOD, "if", EXPR_PAR)
278
+ end
279
+
280
+ def test_ruby21_rational_imaginary_literal
281
+ setup_lexer_class RubyParser::V21
282
+
283
+ assert_lex3 "1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), EXPR_NUM
284
+ assert_lex3 "0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), EXPR_NUM
285
+ assert_lex3 "0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), EXPR_NUM
286
+ assert_lex3 "0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), EXPR_NUM
287
+ assert_lex3 "0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), EXPR_NUM
288
+ assert_lex3 "1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), EXPR_NUM
289
+ assert_lex3 "15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), EXPR_NUM
290
+ assert_lex3 "15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), EXPR_NUM
291
+ assert_lex3 "1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), EXPR_NUM
292
+ assert_lex3 "1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), EXPR_NUM
293
+
294
+ assert_lex3("-10ri", nil,
295
+ :tUMINUS_NUM, "-", EXPR_BEG,
296
+ :tIMAGINARY, Complex(0, Rational(10)), EXPR_NUM)
297
+ end
298
+
299
+ def test_ruby21_rational_literal
300
+ setup_lexer_class RubyParser::V21
301
+
302
+ assert_lex3("10r", nil, :tRATIONAL, Rational(10), EXPR_NUM)
303
+ assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), EXPR_NUM)
304
+ assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), EXPR_NUM)
305
+ assert_lex3("0or", nil, :tRATIONAL, Rational(0), EXPR_NUM)
306
+ assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), EXPR_NUM)
307
+ assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), EXPR_NUM)
308
+ assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), EXPR_NUM)
309
+ assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), EXPR_NUM)
310
+ assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), EXPR_NUM)
311
+ assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), EXPR_NUM)
312
+
313
+ r10 = Rational(10)
314
+ assert_lex3("-10r", nil,
315
+ :tUMINUS_NUM, "-", EXPR_BEG,
316
+ :tRATIONAL, r10, EXPR_NUM)
317
+ end
318
+
319
+ def test_unicode_ident
320
+ s = "@\u1088\u1077\u1093\u1072"
321
+ assert_lex3(s.dup, nil, :tIVAR, s.dup, EXPR_END)
322
+ end
323
+
324
+ def test_why_does_ruby_hate_me?
325
+ assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
326
+ nil,
327
+ :tSTRING, %W[ Nl% \u0000 \u0000 A \u0000 999 ].join, EXPR_END)
328
+ end
329
+
221
330
  def test_yylex_ambiguous_uminus
222
331
  assert_lex3("m -3",
223
332
  nil,
224
- :tIDENTIFIER, "m", :expr_cmdarg,
225
- :tUMINUS_NUM, "-", :expr_beg,
226
- :tINTEGER, 3, :expr_end)
333
+ :tIDENTIFIER, "m", EXPR_CMDARG,
334
+ :tUMINUS_NUM, "-", EXPR_BEG,
335
+ :tINTEGER, 3, EXPR_NUM)
227
336
 
228
337
  # TODO: verify warning
229
338
  end
@@ -231,36 +340,45 @@ class TestRubyLexer < Minitest::Test
231
340
  def test_yylex_ambiguous_uplus
232
341
  assert_lex3("m +3",
233
342
  nil,
234
- :tIDENTIFIER, "m", :expr_cmdarg,
235
- :tINTEGER, 3, :expr_end)
343
+ :tIDENTIFIER, "m", EXPR_CMDARG,
344
+ :tINTEGER, 3, EXPR_NUM)
236
345
 
237
346
  # TODO: verify warning
238
347
  end
239
348
 
240
349
  def test_yylex_and
241
- assert_lex3("&", nil, :tAMPER, "&", :expr_beg)
350
+ assert_lex3("&", nil, :tAMPER, "&", EXPR_BEG)
242
351
  end
243
352
 
244
353
  def test_yylex_and2
245
- assert_lex3("&&", nil, :tANDOP, "&&", :expr_beg)
354
+ assert_lex3("&&", nil, :tANDOP, "&&", EXPR_BEG)
246
355
  end
247
356
 
248
357
  def test_yylex_and2_equals
249
- assert_lex3("&&=", nil, :tOP_ASGN, "&&", :expr_beg)
358
+ assert_lex3("&&=", nil, :tOP_ASGN, "&&", EXPR_BEG)
359
+ end
360
+
361
+ def test_yylex_and_arg
362
+ self.lex_state = EXPR_ARG
363
+
364
+ assert_lex3(" &y",
365
+ nil,
366
+ :tAMPER, "&", EXPR_BEG,
367
+ :tIDENTIFIER, "y", EXPR_ARG)
250
368
  end
251
369
 
252
370
  def test_yylex_and_dot
253
371
  setup_lexer_class RubyParser::V23
254
372
 
255
- assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
373
+ assert_lex3("&.", nil, :tLONELY, "&.", EXPR_DOT)
256
374
  end
257
375
 
258
376
  def test_yylex_and_dot_call
259
377
  setup_lexer_class RubyParser::V23
260
378
 
261
379
  assert_lex3("x&.y", nil,
262
- :tIDENTIFIER, "x", :expr_cmdarg,
263
- :tLONELY, "&.", :expr_dot,
380
+ :tIDENTIFIER, "x", EXPR_CMDARG,
381
+ :tLONELY, "&.", EXPR_DOT,
264
382
  :tIDENTIFIER, "y")
265
383
  end
266
384
 
@@ -268,32 +386,23 @@ class TestRubyLexer < Minitest::Test
268
386
  setup_lexer_class Ruby23Parser
269
387
 
270
388
  assert_lex3("x\n&.y", nil,
271
- :tIDENTIFIER, "x", :expr_cmdarg,
272
- :tLONELY, "&.", :expr_dot,
389
+ :tIDENTIFIER, "x", EXPR_CMDARG,
390
+ :tLONELY, "&.", EXPR_DOT,
273
391
  :tIDENTIFIER, "y")
274
392
  end
275
393
 
276
- def test_yylex_and_arg
277
- self.lex_state = :expr_arg
278
-
279
- assert_lex3(" &y",
280
- nil,
281
- :tAMPER, "&", :expr_beg,
282
- :tIDENTIFIER, "y", :expr_arg)
283
- end
284
-
285
394
  def test_yylex_and_equals
286
- assert_lex3("&=", nil, :tOP_ASGN, "&", :expr_beg)
395
+ assert_lex3("&=", nil, :tOP_ASGN, "&", EXPR_BEG)
287
396
  end
288
397
 
289
398
  def test_yylex_and_expr
290
- self.lex_state = :expr_arg
399
+ self.lex_state = EXPR_ARG
291
400
 
292
401
  assert_lex3("x & y",
293
402
  nil,
294
- :tIDENTIFIER, "x", :expr_cmdarg,
295
- :tAMPER2, "&", :expr_beg,
296
- :tIDENTIFIER, "y", :expr_arg)
403
+ :tIDENTIFIER, "x", EXPR_CMDARG,
404
+ :tAMPER2, "&", EXPR_BEG,
405
+ :tIDENTIFIER, "y", EXPR_ARG)
297
406
  end
298
407
 
299
408
  def test_yylex_and_meth
@@ -301,266 +410,224 @@ class TestRubyLexer < Minitest::Test
301
410
  end
302
411
 
303
412
  def test_yylex_assoc
304
- assert_lex3("=>", nil, :tASSOC, "=>", :expr_beg)
413
+ assert_lex3 "=>", nil, :tASSOC, "=>", EXPR_BEG
305
414
  end
306
415
 
307
- def test_yylex_label__18
308
- setup_lexer_class RubyParser::V18
309
-
310
- assert_lex3("{a:",
416
+ def test_yylex_back_ref
417
+ assert_lex3("[$&, $`, $', $+]",
311
418
  nil,
312
- :tLBRACE, "{", :expr_beg,
313
- :tIDENTIFIER, "a", :expr_arg,
314
- :tSYMBEG, ":", :expr_fname)
419
+ :tLBRACK, "[", EXPR_PAR,
420
+ :tBACK_REF, :&, EXPR_END, :tCOMMA, ",", EXPR_PAR,
421
+ :tBACK_REF, :"`", EXPR_END, :tCOMMA, ",", EXPR_PAR,
422
+ :tBACK_REF, :"'", EXPR_END, :tCOMMA, ",", EXPR_PAR,
423
+ :tBACK_REF, :+, EXPR_END,
424
+ :tRBRACK, "]", EXPR_END)
315
425
  end
316
426
 
317
- def test_yylex_label_in_params__18
318
- setup_lexer_class RubyParser::V18
319
-
320
- assert_lex3("foo(a:",
427
+ def test_yylex_backslash
428
+ assert_lex3("1 \\\n+ 2",
321
429
  nil,
322
- :tIDENTIFIER, "foo", :expr_cmdarg,
323
- :tLPAREN2, "(", :expr_beg,
324
- :tIDENTIFIER, "a", :expr_cmdarg,
325
- :tSYMBEG, ":", :expr_fname)
430
+ :tINTEGER, 1, EXPR_NUM,
431
+ :tPLUS, "+", EXPR_BEG,
432
+ :tINTEGER, 2, EXPR_NUM)
326
433
  end
327
434
 
328
- def test_yylex_label__19
329
- setup_lexer_class RubyParser::V19
330
-
331
- assert_lex3("{a:",
332
- nil,
333
- :tLBRACE, "{", :expr_beg,
334
- :tLABEL, "a", :expr_labelarg)
435
+ def test_yylex_backslash_bad
436
+ refute_lex("1 \\ + 2", :tINTEGER, 1)
335
437
  end
336
438
 
337
- def test_yylex_label_in_params__19
338
- setup_lexer_class RubyParser::V19
339
-
340
- assert_lex3("foo(a:",
439
+ def test_yylex_backtick
440
+ assert_lex3("`ls`",
341
441
  nil,
342
- :tIDENTIFIER, "foo", :expr_cmdarg,
343
- :tLPAREN2, "(", :expr_beg,
344
- :tLABEL, "a", :expr_labelarg)
442
+ :tXSTRING_BEG, "`", EXPR_BEG,
443
+ :tSTRING_CONTENT, "ls", EXPR_BEG,
444
+ :tSTRING_END, "`", EXPR_LIT)
345
445
  end
346
446
 
347
- def test_yylex_paren_string_parens_interpolated
348
- setup_lexer('%((#{b}#{d}))',
349
- s(:dstr,
350
- "(",
351
- s(:evstr, s(:call, nil, :b)),
352
- s(:evstr, s(:call, nil, :d)),
353
- s(:str, ")")))
447
+ def test_yylex_backtick_cmdarg
448
+ self.lex_state = EXPR_DOT
354
449
 
355
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
356
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
357
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
450
+ # \n ensures expr_cmd (TODO: why?)
451
+ assert_lex3("\n`", nil, :tBACK_REF2, "`", EXPR_CMDARG)
452
+ end
358
453
 
359
- emulate_string_interpolation do
360
- assert_next_lexeme :tIDENTIFIER, "b", :expr_arg, 0, 0
361
- end
454
+ def test_yylex_backtick_dot
455
+ self.lex_state = EXPR_DOT
362
456
 
363
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
457
+ assert_lex3("a.`(3)",
458
+ nil,
459
+ :tIDENTIFIER, "a", EXPR_CMDARG,
460
+ :tDOT, ".", EXPR_DOT,
461
+ :tBACK_REF2, "`", EXPR_ARG,
462
+ :tLPAREN2, "(", EXPR_PAR,
463
+ :tINTEGER, 3, EXPR_NUM,
464
+ :tRPAREN, ")", EXPR_ENDFN)
465
+ end
364
466
 
365
- emulate_string_interpolation do
366
- assert_next_lexeme :tIDENTIFIER, "d", :expr_arg, 0, 0
367
- end
467
+ def test_yylex_backtick_method
468
+ self.lex_state = EXPR_FNAME
368
469
 
369
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
370
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
470
+ assert_lex3("`",
471
+ nil,
472
+ :tBACK_REF2, "`", EXPR_END)
473
+ end
371
474
 
372
- refute_lexeme
475
+ def test_yylex_bad_char
476
+ refute_lex(" \010 ")
373
477
  end
374
478
 
375
- def test_yylex_paren_string_interpolated_regexp
376
- setup_lexer('%( #{(/abcd/)} )',
377
- s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
479
+ def test_yylex_bang
480
+ assert_lex3("!", nil, :tBANG, "!", EXPR_BEG)
481
+ end
378
482
 
379
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
380
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
381
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
483
+ def test_yylex_bang_equals
484
+ assert_lex3("!=", nil, :tNEQ, "!=", EXPR_BEG)
485
+ end
382
486
 
383
- emulate_string_interpolation do
384
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
385
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
386
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
387
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
388
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
389
- end
487
+ def test_yylex_bang_tilde
488
+ assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
489
+ end
390
490
 
391
- assert_next_lexeme :tSTRING_CONTENT, " ", :expr_beg, 0, 0
392
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
491
+ def test_yylex_bdot2
492
+ assert_lex3("..42",
493
+ nil, # TODO: s(:dot2, nil, s(:lit, 42)),
393
494
 
394
- refute_lexeme
495
+ :tBDOT2, "..", EXPR_BEG,
496
+ :tINTEGER, 42, EXPR_END|EXPR_ENDARG)
395
497
  end
396
498
 
397
- def test_yylex_not_at_defn__20
398
- setup_lexer_class RubyParser::V20
399
-
400
- assert_lex("def +@; end",
401
- s(:defn, :+@, s(:args), s(:nil)),
499
+ def test_yylex_bdot3
500
+ assert_lex3("...42",
501
+ nil, # TODO: s(:dot2, nil, s(:lit, 42)),
402
502
 
403
- :kDEF, "def", :expr_fname, 0, 0,
404
- :tUPLUS, "+@", :expr_arg, 0, 0,
405
- :tSEMI, ";", :expr_beg, 0, 0,
406
- :kEND, "end", :expr_end, 0, 0)
503
+ :tBDOT3, "...", EXPR_BEG,
504
+ :tINTEGER, 42, EXPR_END|EXPR_ENDARG)
505
+ end
407
506
 
408
- assert_lex("def !@; end",
409
- s(:defn, :"!@", s(:args), s(:nil)),
507
+ def test_yylex_block_bug_1
508
+ assert_lex3("a do end",
509
+ s(:iter, s(:call, nil, :a), 0),
410
510
 
411
- :kDEF, "def", :expr_fname, 0, 0,
412
- :tUBANG, "!@", :expr_arg, 0, 0,
413
- :tSEMI, ";", :expr_beg, 0, 0,
414
- :kEND, "end", :expr_end, 0, 0)
511
+ :tIDENTIFIER, "a", EXPR_CMDARG,
512
+ :kDO, "do", EXPR_BEG,
513
+ :kEND, "end", EXPR_END)
415
514
  end
416
515
 
417
- def test_yylex_not_at_ivar
418
- assert_lex("!@ivar",
419
- s(:call, s(:ivar, :@ivar), :"!"),
516
+ def test_yylex_block_bug_2
517
+ assert_lex3("a = 1\na do\nend",
518
+ s(:block,
519
+ s(:lasgn, :a, s(:lit, 1)),
520
+ s(:iter, s(:call, nil, :a), 0)),
420
521
 
421
- :tBANG, "!", :expr_beg, 0, 0,
422
- :tIVAR, "@ivar", :expr_end, 0, 0)
522
+ :tIDENTIFIER, "a", EXPR_CMDARG,
523
+ :tEQL, "=", EXPR_BEG,
524
+ :tINTEGER, 1, EXPR_NUM,
525
+ :tNL, nil, EXPR_BEG,
526
+ :tIDENTIFIER, "a", EXPR_CMDARG,
527
+ :kDO, "do", EXPR_BEG,
528
+ :kEND, "end", EXPR_END)
423
529
  end
424
530
 
425
- def test_yylex_number_times_ident_times_return_number
426
- assert_lex("1 * b * 3",
427
- s(:call,
428
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
429
- :*, s(:lit, 3)),
531
+ def test_yylex_block_bug_3
532
+ assert_lex3("a { }",
533
+ s(:iter, s(:call, nil, :a), 0),
430
534
 
431
- :tINTEGER, 1, :expr_end, 0, 0,
432
- :tSTAR2, "*", :expr_beg, 0, 0,
433
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
434
- :tSTAR2, "*", :expr_beg, 0, 0,
435
- :tINTEGER, 3, :expr_end, 0, 0)
535
+ :tIDENTIFIER, "a", EXPR_CMDARG, # verified
536
+ :tLCURLY, "{", EXPR_PAR,
537
+ :tRCURLY, "}", EXPR_END)
538
+ end
436
539
 
437
- assert_lex("1 * b *\n 3",
438
- s(:call,
439
- s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
440
- :*, s(:lit, 3)),
540
+ def test_yylex_carat
541
+ assert_lex3("^", nil, :tCARET, "^", EXPR_BEG)
542
+ end
441
543
 
442
- :tINTEGER, 1, :expr_end, 0, 0,
443
- :tSTAR2, "*", :expr_beg, 0, 0,
444
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
445
- :tSTAR2, "*", :expr_beg, 0, 0,
446
- :tINTEGER, 3, :expr_end, 0, 0)
544
+ def test_yylex_carat_equals
545
+ assert_lex3("^=", nil, :tOP_ASGN, "^", EXPR_BEG)
447
546
  end
448
547
 
449
- def test_yylex_paren_string_parens_interpolated_regexp
450
- setup_lexer('%((#{(/abcd/)}))',
451
- s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
548
+ def test_yylex_colon2
549
+ assert_lex3("A::B",
550
+ nil,
551
+ :tCONSTANT, "A", EXPR_CMDARG,
552
+ :tCOLON2, "::", EXPR_DOT,
553
+ :tCONSTANT, "B", EXPR_ARG)
554
+ end
452
555
 
453
- assert_next_lexeme :tSTRING_BEG, "%)", :expr_beg, 0, 0
454
- assert_next_lexeme :tSTRING_CONTENT, "(", :expr_beg, 0, 0
556
+ def test_yylex_colon2_argh
557
+ assert_lex3("module X::Y\n c\nend",
558
+ nil,
559
+ :kMODULE, "module", EXPR_BEG,
560
+ :tCONSTANT, "X", EXPR_CMDARG,
561
+ :tCOLON2, "::", EXPR_DOT,
562
+ :tCONSTANT, "Y", EXPR_ARG,
563
+ :tNL, nil, EXPR_BEG,
564
+ :tIDENTIFIER, "c", EXPR_CMDARG,
565
+ :tNL, nil, EXPR_BEG,
566
+ :kEND, "end", EXPR_END)
567
+ end
455
568
 
456
- assert_next_lexeme :tSTRING_DBEG, nil, :expr_beg, 0, 0
569
+ def test_yylex_colon3
570
+ assert_lex3("::Array",
571
+ nil,
572
+ :tCOLON3, "::", EXPR_BEG,
573
+ :tCONSTANT, "Array", EXPR_ARG)
574
+ end
457
575
 
458
- emulate_string_interpolation do
459
- assert_next_lexeme :tLPAREN, "(", :expr_beg, 1, 0
460
- assert_next_lexeme :tREGEXP_BEG, "/", :expr_beg, 1, 0
461
- assert_next_lexeme :tSTRING_CONTENT, "abcd", :expr_beg, 1, 0
462
- assert_next_lexeme :tREGEXP_END, "", :expr_end, 1, 0
463
- assert_next_lexeme :tRPAREN, ")", :expr_endfn, 0, 0
464
- end
576
+ def test_yylex_comma
577
+ assert_lex3(",", nil, :tCOMMA, ",", EXPR_PAR)
578
+ end
465
579
 
466
- assert_next_lexeme :tSTRING_CONTENT, ")", :expr_beg, 0, 0
467
- assert_next_lexeme :tSTRING_END, ")", :expr_end, 0, 0
580
+ def test_yylex_comment
581
+ assert_lex3("1 # one\n# two\n2",
582
+ nil,
583
+ :tINTEGER, 1, EXPR_NUM,
584
+ :tNL, nil, EXPR_BEG,
585
+ :tINTEGER, 2, EXPR_NUM)
468
586
 
469
- refute_lexeme
587
+ assert_equal "# one\n# two\n", @lex.comments
470
588
  end
471
589
 
472
- def test_yylex_method_parens_chevron
473
- assert_lex("a()<<1",
474
- s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
475
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
476
- :tLPAREN2, "(", :expr_beg, 1, 0,
477
- :tRPAREN, ")", :expr_endfn, 0, 0,
478
- :tLSHFT, "<<" , :expr_beg, 0, 0,
479
- :tINTEGER, 1, :expr_end, 0, 0)
480
- end
590
+ def test_yylex_comment_begin
591
+ assert_lex3("=begin\nblah\nblah\n=end\n42",
592
+ nil,
593
+ :tINTEGER, 42, EXPR_NUM)
481
594
 
482
- def test_yylex_lambda_args__20
483
- setup_lexer_class RubyParser::V20
595
+ assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
596
+ end
484
597
 
485
- assert_lex("-> (a) { }",
486
- s(:iter, s(:call, nil, :lambda),
487
- s(:args, :a)),
598
+ def test_yylex_comment_begin_bad
599
+ refute_lex("=begin\nblah\nblah\n")
488
600
 
489
- :tLAMBDA, nil, :expr_endfn, 0, 0,
490
- :tLPAREN2, "(", :expr_beg, 1, 0,
491
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
492
- :tRPAREN, ")", :expr_endfn, 0, 0,
493
- :tLCURLY, "{", :expr_beg, 0, 1,
494
- :tRCURLY, "}", :expr_endarg, 0, 0)
601
+ assert_equal "", @lex.comments
495
602
  end
496
603
 
497
- def test_yylex_lambda_as_args_with_block__20
498
- setup_lexer_class RubyParser::V20
499
-
500
- assert_lex3("a -> do end do end",
604
+ def test_yylex_comment_begin_not_comment
605
+ assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
501
606
  nil,
502
- :tIDENTIFIER, "a", :expr_cmdarg,
503
- :tLAMBDA, nil, :expr_endfn,
504
- :kDO, "do", :expr_beg,
505
- :kEND, "end", :expr_end,
506
- :kDO_BLOCK, "do", :expr_beg,
507
- :kEND, "end", :expr_end)
607
+ :tIDENTIFIER, "beginfoo", EXPR_CMDARG,
608
+ :tEQL, "=", EXPR_BEG,
609
+ :tINTEGER, 5, EXPR_NUM,
610
+ :tNL, nil, EXPR_BEG,
611
+ :tIDENTIFIER, "p", EXPR_CMDARG,
612
+ :tIDENTIFIER, "x", EXPR_ARG,
613
+ :tEQL, "=", EXPR_BEG,
614
+ :tIDENTIFIER, "beginfoo", EXPR_ARG)
508
615
  end
509
616
 
510
- def test_yylex_lambda_args_opt__20
511
- setup_lexer_class RubyParser::V20
512
-
513
- assert_lex("-> (a=nil) { }",
514
- s(:iter, s(:call, nil, :lambda),
515
- s(:args, s(:lasgn, :a, s(:nil)))),
617
+ def test_yylex_comment_begin_space
618
+ assert_lex3("=begin blah\nblah\n=end\n", nil)
516
619
 
517
- :tLAMBDA, nil, :expr_endfn, 0, 0,
518
- :tLPAREN2, "(", :expr_beg, 1, 0,
519
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
520
- :tEQL, "=", :expr_beg, 1, 0,
521
- :kNIL, "nil", :expr_end, 1, 0,
522
- :tRPAREN, ")", :expr_endfn, 0, 0,
523
- :tLCURLY, "{", :expr_beg, 0, 1,
524
- :tRCURLY, "}", :expr_endarg, 0, 0)
620
+ assert_equal "=begin blah\nblah\n=end\n", @lex.comments
525
621
  end
526
622
 
527
- def test_yylex_lambda_hash__20
528
- setup_lexer_class RubyParser::V20
529
-
530
- assert_lex("-> (a={}) { }",
531
- s(:iter, s(:call, nil, :lambda),
532
- s(:args, s(:lasgn, :a, s(:hash)))),
623
+ def test_yylex_comment_end_space_and_text
624
+ assert_lex3("=begin blah\nblah\n=end blab\n", nil)
533
625
 
534
- :tLAMBDA, nil, :expr_endfn, 0, 0,
535
- :tLPAREN2, "(", :expr_beg, 1, 0,
536
- :tIDENTIFIER, "a", :expr_arg, 1, 0,
537
- :tEQL, "=", :expr_beg, 1, 0,
538
- :tLBRACE, "{", :expr_beg, 1, 1,
539
- :tRCURLY, "}", :expr_endarg, 1, 0,
540
- :tRPAREN, ")", :expr_endfn, 0, 0,
541
- :tLCURLY, "{", :expr_beg, 0, 1,
542
- :tRCURLY, "}", :expr_endarg, 0, 0)
626
+ assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
543
627
  end
544
628
 
545
- def test_yylex_iter_array_curly
546
- assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
547
- s(:iter,
548
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
549
- s(:args, :c, :d)),
550
-
551
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 0,
552
- :tSYMBOL, "a", :expr_end, 0, 0,
553
- :tCOMMA, ",", :expr_beg, 0, 0,
554
- :tLBRACK, "[", :expr_beg, 1, 0,
555
- :tSYMBOL, "b", :expr_end, 1, 0,
556
- :tRBRACK, "]", :expr_endarg, 0, 0,
557
- :tLBRACE_ARG, "{", :expr_beg, 0, 1,
558
- :tPIPE, "|", :expr_beg, 0, 1,
559
- :tIDENTIFIER, "c", :expr_arg, 0, 1,
560
- :tCOMMA, ",", :expr_beg, 0, 1,
561
- :tIDENTIFIER, "d", :expr_arg, 0, 1,
562
- :tPIPE, "|", :expr_beg, 0, 1,
563
- :tRCURLY, "}", :expr_endarg, 0, 0)
629
+ def test_yylex_comment_eos
630
+ assert_lex3("# comment", nil)
564
631
  end
565
632
 
566
633
  def test_yylex_const_call_same_name
@@ -572,18 +639,18 @@ class TestRubyLexer < Minitest::Test
572
639
  0,
573
640
  s(:call, nil, :f, s(:lit, :c)))),
574
641
 
575
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
576
- :tEQL, "=", :expr_beg, 0, 0,
577
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
578
- :tLCURLY, "{", :expr_beg, 0, 1,
579
- :tRCURLY, "}", :expr_endarg, 0, 0,
580
- :tSEMI, ";", :expr_beg, 0, 0,
642
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
643
+ :tEQL, "=", EXPR_BEG, 0, 0,
644
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
645
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
646
+ :tRCURLY, "}", EXPR_END, 0, 0,
647
+ :tSEMI, ";", EXPR_BEG, 0, 0,
581
648
 
582
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
583
- :tLCURLY, "{", :expr_beg, 0, 1,
584
- :tIDENTIFIER, "f", :expr_cmdarg, 0, 1, # different
585
- :tSYMBOL, "c", :expr_end, 0, 1,
586
- :tRCURLY, "}", :expr_endarg, 0, 0)
649
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
650
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
651
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 1, # different
652
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
653
+ :tRCURLY, "}", EXPR_END, 0, 0)
587
654
 
588
655
  assert_lex("X = a { }; b { X :c }",
589
656
  s(:block,
@@ -593,309 +660,103 @@ class TestRubyLexer < Minitest::Test
593
660
  0,
594
661
  s(:call, nil, :X, s(:lit, :c)))),
595
662
 
596
- :tCONSTANT, "X", :expr_cmdarg, 0, 0,
597
- :tEQL, "=", :expr_beg, 0, 0,
598
- :tIDENTIFIER, "a", :expr_arg, 0, 0,
599
- :tLCURLY, "{", :expr_beg, 0, 1,
600
- :tRCURLY, "}", :expr_endarg, 0, 0,
601
- :tSEMI, ";", :expr_beg, 0, 0,
663
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 0,
664
+ :tEQL, "=", EXPR_BEG, 0, 0,
665
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0,
666
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
667
+ :tRCURLY, "}", EXPR_END, 0, 0,
668
+ :tSEMI, ";", EXPR_BEG, 0, 0,
602
669
 
603
- :tIDENTIFIER, "b", :expr_cmdarg, 0, 0,
604
- :tLCURLY, "{", :expr_beg, 0, 1,
605
- :tCONSTANT, "X", :expr_cmdarg, 0, 1, # same
606
- :tSYMBOL, "c", :expr_end, 0, 1,
607
- :tRCURLY, "}", :expr_endarg, 0, 0)
670
+ :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0,
671
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
672
+ :tCONSTANT, "X", EXPR_CMDARG, 0, 1, # same
673
+ :tSYMBOL, "c", EXPR_LIT, 0, 1,
674
+ :tRCURLY, "}", EXPR_END, 0, 0)
608
675
  end
609
676
 
610
- def test_yylex_lasgn_call_same_name
611
- assert_lex("a = b.c :d => 1",
612
- s(:lasgn, :a,
613
- s(:call, s(:call, nil, :b), :c,
614
- s(:hash, s(:lit, :d), s(:lit, 1)))),
615
-
616
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
617
- :tEQL, "=", :expr_beg, 0, 0,
618
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
619
- :tDOT, ".", :expr_dot, 0, 0,
620
- :tIDENTIFIER, "c", :expr_arg, 0, 0, # different
621
- :tSYMBOL, "d", :expr_end, 0, 0,
622
- :tASSOC, "=>", :expr_beg, 0, 0,
623
- :tINTEGER, 1, :expr_end, 0, 0)
624
-
625
- assert_lex("a = b.a :d => 1",
626
- s(:lasgn, :a,
627
- s(:call, s(:call, nil, :b), :a,
628
- s(:hash, s(:lit, :d), s(:lit, 1)))),
629
-
630
- :tIDENTIFIER, "a", :expr_cmdarg, 0, 0,
631
- :tEQL, "=", :expr_beg, 0, 0,
632
- :tIDENTIFIER, "b", :expr_arg, 0, 0,
633
- :tDOT, ".", :expr_dot, 0, 0,
634
- :tIDENTIFIER, "a", :expr_arg, 0, 0, # same as lvar
635
- :tSYMBOL, "d", :expr_end, 0, 0,
636
- :tASSOC, "=>", :expr_beg, 0, 0,
637
- :tINTEGER, 1, :expr_end, 0, 0)
677
+ def test_yylex_constant
678
+ assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", EXPR_CMDARG)
638
679
  end
639
680
 
640
- def test_yylex_back_ref
641
- assert_lex3("[$&, $`, $', $+]",
681
+ def test_yylex_constant_semi
682
+ assert_lex3("ArgumentError;",
642
683
  nil,
643
- :tLBRACK, "[", :expr_beg,
644
- :tBACK_REF, :&, :expr_end, :tCOMMA, ",", :expr_beg,
645
- :tBACK_REF, :"`", :expr_end, :tCOMMA, ",", :expr_beg,
646
- :tBACK_REF, :"'", :expr_end, :tCOMMA, ",", :expr_beg,
647
- :tBACK_REF, :+, :expr_end,
648
- :tRBRACK, "]", :expr_endarg)
684
+ :tCONSTANT, "ArgumentError", EXPR_CMDARG,
685
+ :tSEMI, ";", EXPR_BEG)
649
686
  end
650
687
 
651
- def test_yylex_backslash
652
- assert_lex3("1 \\\n+ 2",
653
- nil,
654
- :tINTEGER, 1, :expr_end,
655
- :tPLUS, "+", :expr_beg,
656
- :tINTEGER, 2, :expr_end)
688
+ def test_yylex_cvar
689
+ assert_lex3("@@blah", nil, :tCVAR, "@@blah", EXPR_END)
657
690
  end
658
691
 
659
- def test_yylex_backslash_bad
660
- refute_lex("1 \\ + 2", :tINTEGER, 1)
661
- end
662
-
663
- def test_yylex_backtick
664
- assert_lex3("`ls`",
665
- nil,
666
- :tXSTRING_BEG, "`", :expr_beg,
667
- :tSTRING_CONTENT, "ls", :expr_beg,
668
- :tSTRING_END, "`", :expr_end)
669
- end
670
-
671
- def test_yylex_backtick_cmdarg
672
- self.lex_state = :expr_dot
673
-
674
- # \n ensures expr_cmd (TODO: why?)
675
- assert_lex3("\n`", nil, :tBACK_REF2, "`", :expr_cmdarg)
676
- end
677
-
678
- def test_yylex_backtick_dot
679
- self.lex_state = :expr_dot
680
-
681
- assert_lex3("a.`(3)",
682
- nil,
683
- :tIDENTIFIER, "a", :expr_cmdarg,
684
- :tDOT, ".", :expr_dot,
685
- :tBACK_REF2, "`", :expr_arg,
686
- :tLPAREN2, "(", :expr_beg,
687
- :tINTEGER, 3, :expr_end,
688
- :tRPAREN, ")", :expr_endfn)
689
- end
690
-
691
- def test_yylex_backtick_method
692
- self.lex_state = :expr_fname
693
-
694
- assert_lex3("`",
695
- nil,
696
- :tBACK_REF2, "`", :expr_end)
697
- end
698
-
699
- def test_yylex_bad_char
700
- refute_lex(" \010 ")
701
- end
702
-
703
- def test_yylex_bang
704
- assert_lex3("!", nil, :tBANG, "!", :expr_beg)
705
- end
706
-
707
- def test_yylex_bang_equals
708
- assert_lex3("!=", nil, :tNEQ, "!=", :expr_beg)
709
- end
710
-
711
- def test_yylex_bang_tilde
712
- assert_lex3("!~", nil, :tNMATCH, "!~", :expr_beg)
713
- end
714
-
715
- def test_yylex_carat
716
- assert_lex3("^", nil, :tCARET, "^", :expr_beg)
717
- end
718
-
719
- def test_yylex_carat_equals
720
- assert_lex3("^=", nil, :tOP_ASGN, "^", :expr_beg)
721
- end
722
-
723
- def test_yylex_colon2
724
- assert_lex3("A::B",
725
- nil,
726
- :tCONSTANT, "A", :expr_cmdarg,
727
- :tCOLON2, "::", :expr_dot,
728
- :tCONSTANT, "B", :expr_arg)
729
- end
730
-
731
- def test_yylex_colon2_argh
732
- assert_lex3("module X::Y\n c\nend",
733
- nil,
734
- :kMODULE, "module", :expr_value,
735
- :tCONSTANT, "X", :expr_arg,
736
- :tCOLON2, "::", :expr_dot,
737
- :tCONSTANT, "Y", :expr_arg,
738
- :tNL, nil, :expr_beg,
739
- :tIDENTIFIER, "c", :expr_cmdarg,
740
- :tNL, nil, :expr_beg,
741
- :kEND, "end", :expr_end)
742
- end
743
-
744
- def test_yylex_colon3
745
- assert_lex3("::Array",
746
- nil,
747
- :tCOLON3, "::", :expr_beg,
748
- :tCONSTANT, "Array", :expr_arg)
749
- end
750
-
751
- def test_yylex_comma
752
- assert_lex3(",", nil, :tCOMMA, ",", :expr_beg)
753
- end
754
-
755
- def test_yylex_comment
756
- assert_lex3("1 # one\n# two\n2",
757
- nil,
758
- :tINTEGER, 1, :expr_end,
759
- :tNL, nil, :expr_beg,
760
- :tINTEGER, 2, :expr_end)
761
-
762
- assert_equal "# one\n# two\n", @lex.comments
763
- end
764
-
765
- def test_yylex_comment_begin
766
- assert_lex3("=begin\nblah\nblah\n=end\n42",
767
- nil,
768
- :tINTEGER, 42, :expr_end)
769
-
770
- assert_equal "=begin\nblah\nblah\n=end\n", @lex.comments
771
- end
772
-
773
- def test_yylex_comment_begin_bad
774
- refute_lex("=begin\nblah\nblah\n")
775
-
776
- assert_equal "", @lex.comments
777
- end
778
-
779
- def test_yylex_comment_begin_not_comment
780
- assert_lex3("beginfoo = 5\np x \\\n=beginfoo",
781
- nil,
782
- :tIDENTIFIER, "beginfoo", :expr_cmdarg,
783
- :tEQL, "=", :expr_beg,
784
- :tINTEGER, 5, :expr_end,
785
- :tNL, nil, :expr_beg,
786
- :tIDENTIFIER, "p", :expr_cmdarg,
787
- :tIDENTIFIER, "x", :expr_arg,
788
- :tEQL, "=", :expr_beg,
789
- :tIDENTIFIER, "beginfoo", :expr_arg)
790
- end
791
-
792
- def test_yylex_comment_begin_space
793
- assert_lex3("=begin blah\nblah\n=end\n", nil)
794
-
795
- assert_equal "=begin blah\nblah\n=end\n", @lex.comments
796
- end
797
-
798
- def test_yylex_comment_end_space_and_text
799
- assert_lex3("=begin blah\nblah\n=end blab\n", nil)
800
-
801
- assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
802
- end
803
-
804
- def test_yylex_comment_eos
805
- assert_lex3("# comment", nil)
806
- end
807
-
808
- def test_yylex_constant
809
- assert_lex3("ArgumentError", nil, :tCONSTANT, "ArgumentError", :expr_cmdarg)
810
- end
811
-
812
- def test_yylex_constant_semi
813
- assert_lex3("ArgumentError;",
814
- nil,
815
- :tCONSTANT, "ArgumentError", :expr_cmdarg,
816
- :tSEMI, ";", :expr_beg)
817
- end
818
-
819
- def test_yylex_cvar
820
- assert_lex3("@@blah", nil, :tCVAR, "@@blah", :expr_end)
821
- end
822
-
823
- def test_yylex_cvar_bad
824
- assert_raises RubyParser::SyntaxError do
825
- assert_lex3("@@1", nil)
826
- end
692
+ def test_yylex_cvar_bad
693
+ assert_raises RubyParser::SyntaxError do
694
+ assert_lex3("@@1", nil)
695
+ end
827
696
  end
828
697
 
829
698
  def test_yylex_def_bad_name
830
- self.lex_state = :expr_fname
831
- refute_lex("def [ ", :kDEF, "def")
699
+ refute_lex3("def [ ",
700
+ :kDEF, "def", EXPR_FNAME)
832
701
  end
833
702
 
834
703
  def test_yylex_div
835
704
  assert_lex3("a / 2",
836
705
  nil,
837
- :tIDENTIFIER, "a", :expr_cmdarg,
838
- :tDIVIDE, "/", :expr_beg,
839
- :tINTEGER, 2, :expr_end)
706
+ :tIDENTIFIER, "a", EXPR_CMDARG,
707
+ :tDIVIDE, "/", EXPR_BEG,
708
+ :tINTEGER, 2, EXPR_NUM)
840
709
  end
841
710
 
842
711
  def test_yylex_div_equals
843
712
  assert_lex3("a /= 2",
844
713
  nil,
845
- :tIDENTIFIER, "a", :expr_cmdarg,
846
- :tOP_ASGN, "/", :expr_beg,
847
- :tINTEGER, 2, :expr_end)
714
+ :tIDENTIFIER, "a", EXPR_CMDARG,
715
+ :tOP_ASGN, "/", EXPR_BEG,
716
+ :tINTEGER, 2, EXPR_NUM)
848
717
  end
849
718
 
850
719
  def test_yylex_do
851
720
  assert_lex3("x do 42 end",
852
721
  nil,
853
- :tIDENTIFIER, "x", :expr_cmdarg,
854
- :kDO, "do", :expr_beg,
855
- :tINTEGER, 42, :expr_end,
856
- :kEND, "end", :expr_end)
722
+ :tIDENTIFIER, "x", EXPR_CMDARG,
723
+ :kDO, "do", EXPR_BEG,
724
+ :tINTEGER, 42, EXPR_NUM,
725
+ :kEND, "end", EXPR_END)
857
726
  end
858
727
 
859
728
  def test_yylex_do_block
860
- self.lex_state = :expr_endarg
729
+ self.lex_state = EXPR_ENDARG
861
730
 
862
731
  assert_lex3("x.y do 42 end",
863
732
  nil,
864
- :tIDENTIFIER, "x", :expr_end,
865
- :tDOT, ".", :expr_dot,
866
- :tIDENTIFIER, "y", :expr_arg,
867
- :kDO_BLOCK, "do", :expr_beg,
868
- :tINTEGER, 42, :expr_end,
869
- :kEND, "end", :expr_end) do
733
+ :tIDENTIFIER, "x", EXPR_END,
734
+ :tDOT, ".", EXPR_DOT,
735
+ :tIDENTIFIER, "y", EXPR_ARG,
736
+ :kDO_BLOCK, "do", EXPR_BEG,
737
+ :tINTEGER, 42, EXPR_NUM,
738
+ :kEND, "end", EXPR_END) do
870
739
  @lex.cmdarg.push true
871
740
  end
872
741
  end
873
742
 
874
743
  def test_yylex_do_block2
875
- self.lex_state = :expr_endarg
744
+ self.lex_state = EXPR_ENDARG
876
745
 
877
746
  assert_lex3("do 42 end",
878
747
  nil,
879
- :kDO_BLOCK, "do", :expr_beg,
880
- :tINTEGER, 42, :expr_end,
881
- :kEND, "end", :expr_end)
882
- end
883
-
884
- def test_yylex_is_your_spacebar_broken?
885
- assert_lex3(":a!=:b",
886
- nil,
887
- :tSYMBOL, "a", :expr_end,
888
- :tNEQ, "!=", :expr_beg,
889
- :tSYMBOL, "b", :expr_end)
748
+ :kDO, "do", EXPR_BEG,
749
+ :tINTEGER, 42, EXPR_NUM,
750
+ :kEND, "end", EXPR_END)
890
751
  end
891
752
 
892
753
  def test_yylex_do_cond
893
754
  assert_lex3("x do 42 end",
894
755
  nil,
895
- :tIDENTIFIER, "x", :expr_cmdarg,
896
- :kDO_COND, "do", :expr_beg,
897
- :tINTEGER, 42, :expr_end,
898
- :kEND, "end", :expr_end) do
756
+ :tIDENTIFIER, "x", EXPR_CMDARG,
757
+ :kDO_COND, "do", EXPR_BEG,
758
+ :tINTEGER, 42, EXPR_NUM,
759
+ :kEND, "end", EXPR_END) do
899
760
  @lex.cond.push true
900
761
  end
901
762
  end
@@ -905,41 +766,53 @@ class TestRubyLexer < Minitest::Test
905
766
  assert_includes(e.message, "is not allowed as a global variable name")
906
767
  end
907
768
 
908
- def test_yylex_dollar_eos
909
- assert_lex3("$", nil, "$", "$", :expr_end) # FIX: wtf is this?!?
910
- end
911
-
912
769
  def test_yylex_dot # HINT message sends
913
- assert_lex3(".", nil, :tDOT, ".", :expr_dot)
770
+ assert_lex3(".", nil, :tDOT, ".", EXPR_DOT)
914
771
  end
915
772
 
916
773
  def test_yylex_dot2
917
- assert_lex3("..", nil, :tDOT2, "..", :expr_beg)
774
+ assert_lex3("1..2",
775
+ s(:lit, 1..2),
776
+
777
+ :tINTEGER, 1, EXPR_END|EXPR_ENDARG,
778
+ :tDOT2, "..", EXPR_BEG,
779
+ :tINTEGER, 2, EXPR_END|EXPR_ENDARG)
780
+
781
+ self.lex_state = EXPR_END|EXPR_ENDARG
782
+ assert_lex3("..", nil, :tDOT2, "..", EXPR_BEG)
918
783
  end
919
784
 
920
785
  def test_yylex_dot3
921
- assert_lex3("...", nil, :tDOT3, "...", :expr_beg)
786
+ assert_lex3("1...2",
787
+ s(:lit, 1...2),
788
+
789
+ :tINTEGER, 1, EXPR_END|EXPR_ENDARG,
790
+ :tDOT3, "...", EXPR_BEG,
791
+ :tINTEGER, 2, EXPR_END|EXPR_ENDARG)
792
+
793
+ self.lex_state = EXPR_END|EXPR_ENDARG
794
+ assert_lex3("...", nil, :tDOT3, "...", EXPR_BEG)
922
795
  end
923
796
 
924
797
  def test_yylex_equals
925
798
  # FIX: this sucks
926
- assert_lex3("=", nil, :tEQL, "=", :expr_beg)
799
+ assert_lex3("=", nil, :tEQL, "=", EXPR_BEG)
927
800
  end
928
801
 
929
802
  def test_yylex_equals2
930
- assert_lex3("==", nil, :tEQ, "==", :expr_beg)
803
+ assert_lex3("==", nil, :tEQ, "==", EXPR_BEG)
931
804
  end
932
805
 
933
806
  def test_yylex_equals3
934
- assert_lex3("===", nil, :tEQQ, "===", :expr_beg)
807
+ assert_lex3("===", nil, :tEQQ, "===", EXPR_BEG)
935
808
  end
936
809
 
937
810
  def test_yylex_equals_tilde
938
- assert_lex3("=~", nil, :tMATCH, "=~", :expr_beg)
811
+ assert_lex3("=~", nil, :tMATCH, "=~", EXPR_BEG)
939
812
  end
940
813
 
941
814
  def test_yylex_float
942
- assert_lex3("1.0", nil, :tFLOAT, 1.0, :expr_end)
815
+ assert_lex3("1.0", nil, :tFLOAT, 1.0, EXPR_NUM)
943
816
  end
944
817
 
945
818
  def test_yylex_float_bad_no_underscores
@@ -957,55 +830,48 @@ class TestRubyLexer < Minitest::Test
957
830
  def test_yylex_float_call
958
831
  assert_lex3("1.0.to_s",
959
832
  nil,
960
- :tFLOAT, 1.0, :expr_end,
961
- :tDOT, ".", :expr_dot,
962
- :tIDENTIFIER, "to_s", :expr_arg)
833
+ :tFLOAT, 1.0, EXPR_NUM,
834
+ :tDOT, ".", EXPR_DOT,
835
+ :tIDENTIFIER, "to_s", EXPR_ARG)
963
836
  end
964
837
 
965
838
  def test_yylex_float_dot_E
966
839
  assert_lex3("1.0E10",
967
840
  nil,
968
- :tFLOAT, 10000000000.0, :expr_end)
841
+ :tFLOAT, 10000000000.0, EXPR_NUM)
969
842
  end
970
843
 
971
844
  def test_yylex_float_dot_E_neg
972
845
  assert_lex3("-1.0E10",
973
846
  nil,
974
- :tUMINUS_NUM, "-", :expr_beg,
975
- :tFLOAT, 10000000000.0, :expr_end)
847
+ :tUMINUS_NUM, "-", EXPR_BEG,
848
+ :tFLOAT, 10000000000.0, EXPR_NUM)
976
849
  end
977
850
 
978
851
  def test_yylex_float_dot_e
979
852
  assert_lex3("1.0e10",
980
853
  nil,
981
- :tFLOAT, 10000000000.0, :expr_end)
854
+ :tFLOAT, 10000000000.0, EXPR_NUM)
982
855
  end
983
856
 
984
857
  def test_yylex_float_dot_e_neg
985
858
  assert_lex3("-1.0e10",
986
859
  nil,
987
- :tUMINUS_NUM, "-", :expr_beg,
988
- :tFLOAT, 10000000000.0, :expr_end)
860
+ :tUMINUS_NUM, "-", EXPR_BEG,
861
+ :tFLOAT, 10000000000.0, EXPR_NUM)
989
862
  end
990
863
 
991
864
  def test_yylex_float_e
992
865
  assert_lex3("1e10",
993
866
  nil,
994
- :tFLOAT, 10000000000.0, :expr_end)
867
+ :tFLOAT, 10000000000.0, EXPR_NUM)
995
868
  end
996
869
 
997
870
  def test_yylex_float_e_bad_double_e
998
871
  assert_lex3("1e2e3",
999
872
  nil,
1000
- :tFLOAT, 100, :expr_end,
1001
- :tIDENTIFIER, "e3", :expr_end)
1002
- end
1003
-
1004
- def test_yylex_float_if_modifier
1005
- assert_lex3("1e2if",
1006
- nil,
1007
- :tFLOAT, 100, :expr_end,
1008
- :kIF_MOD, "if", :expr_beg)
873
+ :tFLOAT, 100, EXPR_NUM,
874
+ :tIDENTIFIER, "e3", EXPR_END)
1009
875
  end
1010
876
 
1011
877
  def test_yylex_float_e_bad_trailing_underscore
@@ -1013,205 +879,288 @@ class TestRubyLexer < Minitest::Test
1013
879
  end
1014
880
 
1015
881
  def test_yylex_float_e_minus
1016
- assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, :expr_end)
882
+ assert_lex3("1e-10", nil, :tFLOAT, 1.0e-10, EXPR_NUM)
1017
883
  end
1018
884
 
1019
885
  def test_yylex_float_e_neg
1020
886
  assert_lex3("-1e10",
1021
887
  nil,
1022
- :tUMINUS_NUM, "-", :expr_beg,
1023
- :tFLOAT, 10000000000.0, :expr_end)
888
+ :tUMINUS_NUM, "-", EXPR_BEG,
889
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1024
890
  end
1025
891
 
1026
892
  def test_yylex_float_e_neg_minus
1027
893
  assert_lex3("-1e-10",
1028
894
  nil,
1029
- :tUMINUS_NUM, "-", :expr_beg,
1030
- :tFLOAT, 1.0e-10, :expr_end)
895
+ :tUMINUS_NUM, "-", EXPR_BEG,
896
+ :tFLOAT, 1.0e-10, EXPR_NUM)
1031
897
  end
1032
898
 
1033
899
  def test_yylex_float_e_neg_plus
1034
900
  assert_lex3("-1e+10",
1035
901
  nil,
1036
- :tUMINUS_NUM, "-", :expr_beg,
1037
- :tFLOAT, 10000000000.0, :expr_end)
902
+ :tUMINUS_NUM, "-", EXPR_BEG,
903
+ :tFLOAT, 10000000000.0, EXPR_NUM)
1038
904
  end
1039
905
 
1040
906
  def test_yylex_float_e_plus
1041
- assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, :expr_end)
907
+ assert_lex3("1e+10", nil, :tFLOAT, 10000000000.0, EXPR_NUM)
1042
908
  end
1043
909
 
1044
910
  def test_yylex_float_e_zero
1045
- assert_lex3("0e0", nil, :tFLOAT, 0.0, :expr_end)
911
+ assert_lex3("0e0", nil, :tFLOAT, 0.0, EXPR_NUM)
912
+ end
913
+
914
+ def test_yylex_float_if_modifier
915
+ assert_lex3("1e2if",
916
+ nil,
917
+ :tFLOAT, 100, EXPR_NUM,
918
+ :kIF_MOD, "if", EXPR_PAR)
1046
919
  end
1047
920
 
1048
921
  def test_yylex_float_neg
1049
922
  assert_lex3("-1.0",
1050
923
  nil,
1051
- :tUMINUS_NUM, "-", :expr_beg,
1052
- :tFLOAT, 1.0, :expr_end)
924
+ :tUMINUS_NUM, "-", EXPR_BEG,
925
+ :tFLOAT, 1.0, EXPR_NUM)
1053
926
  end
1054
927
 
1055
928
  def test_yylex_ge
1056
929
  assert_lex3("a >= 2",
1057
930
  nil,
1058
- :tIDENTIFIER, "a", :expr_cmdarg,
1059
- :tGEQ, ">=", :expr_beg,
1060
- :tINTEGER, 2, :expr_end)
931
+ :tIDENTIFIER, "a", EXPR_CMDARG,
932
+ :tGEQ, ">=", EXPR_BEG,
933
+ :tINTEGER, 2, EXPR_NUM)
1061
934
  end
1062
935
 
1063
936
  def test_yylex_global
1064
- assert_lex3("$blah", nil, :tGVAR, "$blah", :expr_end)
937
+ assert_lex3("$blah", nil, :tGVAR, "$blah", EXPR_END)
1065
938
  end
1066
939
 
1067
940
  def test_yylex_global_backref
1068
- self.lex_state = :expr_fname
941
+ self.lex_state = EXPR_FNAME
1069
942
 
1070
- assert_lex3("$`", nil, :tGVAR, "$`", :expr_end)
943
+ assert_lex3("$`", nil, :tGVAR, "$`", EXPR_END)
1071
944
  end
1072
945
 
1073
946
  def test_yylex_global_dash_nothing
1074
- assert_lex3("$- ", nil, :tGVAR, "$-", :expr_end)
947
+ refute_lex3("$- ", nil) # fails 2.1+
948
+
949
+ setup_lexer_class RubyParser::V20
950
+ assert_lex3("$- ", nil, :tGVAR, "$-", EXPR_END)
1075
951
  end
1076
952
 
1077
953
  def test_yylex_global_dash_something
1078
- assert_lex3("$-x", nil, :tGVAR, "$-x", :expr_end)
954
+ assert_lex3("$-x", nil, :tGVAR, "$-x", EXPR_END)
1079
955
  end
1080
956
 
1081
957
  def test_yylex_global_number
1082
- self.lex_state = :expr_fname
958
+ self.lex_state = EXPR_FNAME
1083
959
 
1084
- assert_lex3("$1", nil, :tGVAR, "$1", :expr_end)
960
+ assert_lex3("$1", nil, :tGVAR, "$1", EXPR_END)
1085
961
  end
1086
962
 
1087
963
  def test_yylex_global_number_big
1088
- self.lex_state = :expr_fname
964
+ self.lex_state = EXPR_FNAME
1089
965
 
1090
- assert_lex3("$1234", nil, :tGVAR, "$1234", :expr_end)
966
+ assert_lex3("$1234", nil, :tGVAR, "$1234", EXPR_END)
967
+ end
968
+
969
+ def test_yylex_global_I_have_no_words
970
+ assert_lex3("$x\xE2\x80\x8B = 42", # zero width space?!?!?
971
+ nil,
972
+ :tGVAR, "$x\xE2\x80\x8B", EXPR_END,
973
+ :tEQL, "=", EXPR_BEG,
974
+ :tINTEGER, 42, EXPR_NUM)
1091
975
  end
1092
976
 
1093
977
  def test_yylex_global_other
1094
978
  assert_lex3("[$~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $\"]",
1095
979
  nil,
1096
- :tLBRACK, "[", :expr_beg,
1097
- :tGVAR, "$~", :expr_end, :tCOMMA, ",", :expr_beg,
1098
- :tGVAR, "$*", :expr_end, :tCOMMA, ",", :expr_beg,
1099
- :tGVAR, "$$", :expr_end, :tCOMMA, ",", :expr_beg,
1100
- :tGVAR, "$?", :expr_end, :tCOMMA, ",", :expr_beg,
1101
- :tGVAR, "$!", :expr_end, :tCOMMA, ",", :expr_beg,
1102
- :tGVAR, "$@", :expr_end, :tCOMMA, ",", :expr_beg,
1103
- :tGVAR, "$/", :expr_end, :tCOMMA, ",", :expr_beg,
1104
- :tGVAR, "$\\", :expr_end, :tCOMMA, ",", :expr_beg,
1105
- :tGVAR, "$;", :expr_end, :tCOMMA, ",", :expr_beg,
1106
- :tGVAR, "$,", :expr_end, :tCOMMA, ",", :expr_beg,
1107
- :tGVAR, "$.", :expr_end, :tCOMMA, ",", :expr_beg,
1108
- :tGVAR, "$=", :expr_end, :tCOMMA, ",", :expr_beg,
1109
- :tGVAR, "$:", :expr_end, :tCOMMA, ",", :expr_beg,
1110
- :tGVAR, "$<", :expr_end, :tCOMMA, ",", :expr_beg,
1111
- :tGVAR, "$>", :expr_end, :tCOMMA, ",", :expr_beg,
1112
- :tGVAR, "$\"", :expr_end,
1113
- :tRBRACK, "]", :expr_endarg)
980
+ :tLBRACK, "[", EXPR_PAR,
981
+ :tGVAR, "$~", EXPR_END, :tCOMMA, ",", EXPR_PAR,
982
+ :tGVAR, "$*", EXPR_END, :tCOMMA, ",", EXPR_PAR,
983
+ :tGVAR, "$$", EXPR_END, :tCOMMA, ",", EXPR_PAR,
984
+ :tGVAR, "$?", EXPR_END, :tCOMMA, ",", EXPR_PAR,
985
+ :tGVAR, "$!", EXPR_END, :tCOMMA, ",", EXPR_PAR,
986
+ :tGVAR, "$@", EXPR_END, :tCOMMA, ",", EXPR_PAR,
987
+ :tGVAR, "$/", EXPR_END, :tCOMMA, ",", EXPR_PAR,
988
+ :tGVAR, "$\\", EXPR_END, :tCOMMA, ",", EXPR_PAR,
989
+ :tGVAR, "$;", EXPR_END, :tCOMMA, ",", EXPR_PAR,
990
+ :tGVAR, "$,", EXPR_END, :tCOMMA, ",", EXPR_PAR,
991
+ :tGVAR, "$.", EXPR_END, :tCOMMA, ",", EXPR_PAR,
992
+ :tGVAR, "$=", EXPR_END, :tCOMMA, ",", EXPR_PAR,
993
+ :tGVAR, "$:", EXPR_END, :tCOMMA, ",", EXPR_PAR,
994
+ :tGVAR, "$<", EXPR_END, :tCOMMA, ",", EXPR_PAR,
995
+ :tGVAR, "$>", EXPR_END, :tCOMMA, ",", EXPR_PAR,
996
+ :tGVAR, "$\"", EXPR_END,
997
+ :tRBRACK, "]", EXPR_END)
1114
998
  end
1115
999
 
1116
1000
  def test_yylex_global_underscore
1117
- assert_lex3("$_", nil, :tGVAR, "$_", :expr_end)
1001
+ assert_lex3("$_", nil, :tGVAR, "$_", EXPR_END)
1118
1002
  end
1119
1003
 
1120
1004
  def test_yylex_global_wierd
1121
- assert_lex3("$__blah", nil, :tGVAR, "$__blah", :expr_end)
1005
+ assert_lex3("$__blah", nil, :tGVAR, "$__blah", EXPR_END)
1122
1006
  end
1123
1007
 
1124
1008
  def test_yylex_global_zero
1125
- assert_lex3("$0", nil, :tGVAR, "$0", :expr_end)
1009
+ assert_lex3("$0", nil, :tGVAR, "$0", EXPR_END)
1126
1010
  end
1127
1011
 
1128
1012
  def test_yylex_gt
1129
1013
  assert_lex3("a > 2",
1130
1014
  nil,
1131
- :tIDENTIFIER, "a", :expr_cmdarg,
1132
- :tGT, ">", :expr_beg,
1133
- :tINTEGER, 2, :expr_end)
1015
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1016
+ :tGT, ">", EXPR_BEG,
1017
+ :tINTEGER, 2, EXPR_NUM)
1018
+ end
1019
+
1020
+ def test_yylex_hash_colon
1021
+ assert_lex("{a:1}",
1022
+ s(:hash, s(:lit, :a), s(:lit, 1)),
1023
+
1024
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1025
+ :tLABEL, "a", EXPR_LAB, 0, 1,
1026
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
1027
+ :tRCURLY, "}", EXPR_END, 0, 0)
1028
+ end
1029
+
1030
+ def test_yylex_hash_colon_double_quoted_symbol
1031
+ assert_lex('{"abc": :b}',
1032
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1033
+
1034
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1035
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1036
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1037
+ :tRCURLY, "}", EXPR_END, 0, 0)
1038
+ end
1039
+
1040
+ def test_yylex_hash_colon_double_quoted_symbol_22
1041
+ setup_lexer_class RubyParser::V22
1042
+
1043
+ assert_lex('{"abc": :b}',
1044
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1045
+
1046
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1047
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1048
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1049
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1050
+ end
1051
+
1052
+ def test_yylex_hash_colon_double_quoted_with_escapes
1053
+ assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
1054
+ nil,
1055
+
1056
+ :tLBRACE, "{", EXPR_PAR,
1057
+ :tLABEL, "s\tr\i\ng\\foo'bar", EXPR_LAB,
1058
+ :tINTEGER, 1, EXPR_NUM,
1059
+ :tRCURLY, "}", EXPR_END)
1060
+ end
1061
+
1062
+ def test_yylex_hash_colon_quoted_22
1063
+ setup_lexer_class RubyParser::V22
1064
+
1065
+ assert_lex("{'a':1}",
1066
+ s(:hash, s(:lit, :a), s(:lit, 1)),
1067
+
1068
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1069
+ :tLABEL, "a", EXPR_LAB, 0, 1,
1070
+ :tINTEGER, 1, EXPR_NUM, 0, 1,
1071
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1072
+ end
1073
+
1074
+ def test_yylex_hash_colon_quoted_symbol
1075
+ assert_lex("{'abc': :b}",
1076
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1077
+
1078
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1079
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1080
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1081
+ :tRCURLY, "}", EXPR_END, 0, 0)
1082
+ end
1083
+
1084
+ def test_yylex_hash_colon_quoted_symbol_22
1085
+ setup_lexer_class RubyParser::V22
1086
+
1087
+ assert_lex("{'abc': :b}",
1088
+ s(:hash, s(:lit, :abc), s(:lit, :b)),
1089
+
1090
+ :tLBRACE, "{", EXPR_PAR, 0, 1,
1091
+ :tLABEL, "abc", EXPR_LAB, 0, 1,
1092
+ :tSYMBOL, "b", EXPR_LIT, 0, 1,
1093
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1094
+ end
1095
+
1096
+ def test_yylex_hash_colon_quoted_with_escapes
1097
+ assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
1098
+ nil,
1099
+
1100
+ :tLBRACE, "{", EXPR_PAR,
1101
+ :tLABEL, "s\\tr\\i\\ng\\foo'bar", EXPR_LAB,
1102
+ :tINTEGER, 1, EXPR_NUM,
1103
+ :tRCURLY, "}", EXPR_END)
1134
1104
  end
1135
1105
 
1136
1106
  def test_yylex_heredoc_backtick
1137
- assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1107
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n",
1138
1108
  nil,
1139
- :tIDENTIFIER, "a", :expr_cmdarg,
1140
- :tEQL, "=", :expr_beg,
1141
- :tXSTRING_BEG, "`", :expr_beg,
1142
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1143
- :tSTRING_END, "EOF", :expr_end,
1144
- :tNL, nil, :expr_beg)
1109
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1110
+ :tEQL, "=", EXPR_BEG,
1111
+ :tXSTRING_BEG, "`", EXPR_BEG,
1112
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1113
+ :tSTRING_END, "EOF", EXPR_LIT,
1114
+ :tNL, nil, EXPR_BEG)
1145
1115
  end
1146
1116
 
1147
1117
  def test_yylex_heredoc_double
1148
1118
  assert_lex3("a = <<\"EOF\"\n blah blah\nEOF\n\n",
1149
1119
  nil,
1150
- :tIDENTIFIER, "a", :expr_cmdarg,
1151
- :tEQL, "=", :expr_beg,
1152
- :tSTRING_BEG, "\"", :expr_beg,
1153
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1154
- :tSTRING_END, "EOF", :expr_end,
1155
- :tNL, nil, :expr_beg)
1120
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1121
+ :tEQL, "=", EXPR_BEG,
1122
+ :tSTRING_BEG, "\"", EXPR_BEG,
1123
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1124
+ :tSTRING_END, "EOF", EXPR_LIT,
1125
+ :tNL, nil, EXPR_BEG)
1156
1126
  end
1157
1127
 
1158
1128
  def test_yylex_heredoc_double_dash
1159
- assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1129
+ assert_lex3("a = \" blah blah\n\".strip\n42",
1160
1130
  nil,
1161
- :tIDENTIFIER, "a", :expr_cmdarg,
1162
- :tEQL, "=", :expr_beg,
1163
- :tSTRING_BEG, "\"", :expr_beg,
1164
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1165
- :tSTRING_END, "EOF", :expr_end,
1166
- :tNL, nil, :expr_beg)
1167
- end
1131
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1132
+ :tEQL, "=", EXPR_BEG,
1133
+ :tSTRING, " blah blah\n", EXPR_END,
1134
+ :tDOT, ".", EXPR_DOT,
1135
+ :tIDENTIFIER, "strip", EXPR_ARG,
1136
+ :tNL, nil, EXPR_BEG,
1168
1137
 
1169
- def test_yylex_heredoc_double_squiggly
1170
- setup_lexer_class Ruby23Parser
1138
+ :tINTEGER, 42, EXPR_END
1139
+ )
1171
1140
 
1172
- assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1141
+ assert_lex3("a = <<-\"EOF\".strip\n blah blah\n EOF\n42",
1173
1142
  nil,
1174
- :tIDENTIFIER, "a", :expr_cmdarg,
1175
- :tEQL, "=", :expr_beg,
1176
- :tSTRING_BEG, "\"", :expr_beg,
1177
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1178
- :tSTRING_END, "EOF", :expr_end,
1179
- :tNL, nil, :expr_beg)
1180
- end
1143
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1144
+ :tEQL, "=", EXPR_BEG,
1145
+ :tSTRING_BEG, "\"", EXPR_BEG,
1146
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1147
+ :tSTRING_END, "EOF", EXPR_LIT,
1181
1148
 
1182
- # mri handles tabs in a pretty specific way:
1183
- # https://github.com/ruby/ruby/blob/trunk/parse.y#L5925
1184
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_remaining
1185
- setup_lexer_class Ruby23Parser
1149
+ :tDOT, ".", EXPR_DOT,
1150
+ :tIDENTIFIER, "strip", EXPR_ARG,
1186
1151
 
1187
- assert_lex3("a = <<~\"EOF\"\n blah blah\n \tblah blah\n EOF\n\n",
1188
- nil,
1189
- :tIDENTIFIER, "a", :expr_cmdarg,
1190
- :tEQL, "=", :expr_beg,
1191
- :tSTRING_BEG, "\"", :expr_beg,
1192
- :tSTRING_CONTENT, "blah blah\n\tblah blah\n", :expr_beg,
1193
- :tSTRING_END, "EOF", :expr_end,
1194
- :tNL, nil, :expr_beg)
1195
- end
1196
-
1197
- def test_yylex_heredoc_double_squiggly_with_tab_indentation_removed
1198
- setup_lexer_class Ruby23Parser
1152
+ :tNL, nil, EXPR_BEG,
1199
1153
 
1200
- assert_lex3("a = <<~\"EOF\"\n blah blah\n\t blah blah\n EOF\n\n",
1201
- nil,
1202
- :tIDENTIFIER, "a", :expr_cmdarg,
1203
- :tEQL, "=", :expr_beg,
1204
- :tSTRING_BEG, "\"", :expr_beg,
1205
- :tSTRING_CONTENT, "blah blah\n blah blah\n", :expr_beg,
1206
- :tSTRING_END, "EOF", :expr_end,
1207
- :tNL, nil, :expr_beg)
1154
+ :tINTEGER, 42, EXPR_END
1155
+ )
1208
1156
  end
1209
1157
 
1210
1158
  def test_yylex_heredoc_double_eos
1211
1159
  refute_lex("a = <<\"EOF\"\nblah",
1212
1160
  :tIDENTIFIER, "a",
1213
1161
  :tEQL, "=",
1214
- :tSTRING_BEG, "\"")
1162
+ :tSTRING_BEG, "\"",
1163
+ :tSTRING_CONTENT, "blah")
1215
1164
  end
1216
1165
 
1217
1166
  def test_yylex_heredoc_double_eos_nl
@@ -1222,41 +1171,56 @@ class TestRubyLexer < Minitest::Test
1222
1171
  end
1223
1172
 
1224
1173
  def test_yylex_heredoc_double_interp
1225
- assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n\n",
1226
- nil,
1227
- :tIDENTIFIER, "a", :expr_cmdarg,
1228
- :tEQL, "=", :expr_beg,
1229
- :tSTRING_BEG, "\"", :expr_beg,
1230
- :tSTRING_CONTENT, "#x a ", :expr_beg,
1231
- :tSTRING_DVAR, "\#@", :expr_beg,
1232
- :tSTRING_CONTENT, "@a b ", :expr_beg, # HUH?
1233
- :tSTRING_DVAR, "\#$", :expr_beg,
1234
- :tSTRING_CONTENT, "$b c ", :expr_beg, # HUH?
1235
- :tSTRING_DBEG, "\#{", :expr_beg,
1236
- :tSTRING_CONTENT, "3} \n", :expr_beg, # HUH?
1237
- :tSTRING_END, "EOF", :expr_end,
1238
- :tNL, nil, :expr_beg)
1174
+ assert_lex3("a = <<\"EOF\"\n#x a \#@a b \#$b c \#@@d \#{3} \nEOF\n\n",
1175
+ nil,
1176
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1177
+ :tEQL, "=", EXPR_BEG,
1178
+ :tSTRING_BEG, "\"", EXPR_BEG,
1179
+ :tSTRING_CONTENT, "#x a ", EXPR_BEG,
1180
+ :tSTRING_DVAR, "#", EXPR_BEG,
1181
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG, # HUH?
1182
+ :tSTRING_DVAR, "#", EXPR_BEG,
1183
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG, # HUH?
1184
+ :tSTRING_DVAR, "#", EXPR_BEG,
1185
+ :tSTRING_CONTENT, "@@d ", EXPR_BEG, # HUH?
1186
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1187
+ :tSTRING_CONTENT, "3} \n", EXPR_BEG,
1188
+ :tSTRING_END, "EOF", EXPR_LIT,
1189
+ :tNL, nil, EXPR_BEG)
1190
+ end
1191
+
1192
+ def test_yylex_heredoc_double_squiggly
1193
+ setup_lexer_class Ruby23Parser
1194
+
1195
+ assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1196
+ nil,
1197
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1198
+ :tEQL, "=", EXPR_BEG,
1199
+ :tSTRING_BEG, "\"", EXPR_BEG,
1200
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1201
+ :tSTRING_END, "EOF", EXPR_LIT,
1202
+ :tNL, nil, EXPR_BEG)
1239
1203
  end
1240
1204
 
1241
1205
  def test_yylex_heredoc_empty
1242
1206
  assert_lex3("<<\"\"\n\#{x}\nblah2\n\n\n",
1243
1207
  nil,
1244
- :tSTRING_BEG, "\"", :expr_beg,
1245
- :tSTRING_DBEG, "\#{", :expr_beg,
1246
- :tSTRING_CONTENT, "x}\nblah2\n", :expr_beg,
1247
- :tSTRING_END, "", :expr_end,
1248
- :tNL, nil, :expr_beg)
1208
+ :tSTRING_BEG, "\"", EXPR_BEG,
1209
+ :tSTRING_DBEG, "\#{", EXPR_BEG,
1210
+ :tSTRING_CONTENT, "x}\nblah2\n", EXPR_BEG,
1211
+ :tSTRING_END, "", EXPR_LIT,
1212
+ :tNL, nil, EXPR_BEG)
1249
1213
  end
1250
1214
 
1251
1215
  def test_yylex_heredoc_none
1252
1216
  assert_lex3("a = <<EOF\nblah\nblah\nEOF\n",
1253
1217
  nil,
1254
- :tIDENTIFIER, "a", :expr_cmdarg,
1255
- :tEQL, "=", :expr_beg,
1256
- :tSTRING_BEG, "\"", :expr_beg,
1257
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1258
- :tSTRING_END, "EOF", :expr_end,
1259
- :tNL, nil, :expr_beg)
1218
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1219
+ :tEQL, "=", EXPR_BEG,
1220
+ :tSTRING_BEG, "\"", EXPR_BEG,
1221
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1222
+ :tSTRING_END, "EOF", EXPR_LIT,
1223
+ :tNL, nil, EXPR_BEG)
1260
1224
  end
1261
1225
 
1262
1226
  def test_yylex_heredoc_none_bad_eos
@@ -1269,12 +1233,12 @@ class TestRubyLexer < Minitest::Test
1269
1233
  def test_yylex_heredoc_none_dash
1270
1234
  assert_lex3("a = <<-EOF\nblah\nblah\n EOF\n",
1271
1235
  nil,
1272
- :tIDENTIFIER, "a", :expr_cmdarg,
1273
- :tEQL, "=", :expr_beg,
1274
- :tSTRING_BEG, "\"", :expr_beg,
1275
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1276
- :tSTRING_END, "EOF", :expr_end,
1277
- :tNL, nil, :expr_beg)
1236
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1237
+ :tEQL, "=", EXPR_BEG,
1238
+ :tSTRING_BEG, "\"", EXPR_BEG,
1239
+ :tSTRING_CONTENT, "blah\nblah\n", EXPR_BEG,
1240
+ :tSTRING_END, "EOF", EXPR_LIT,
1241
+ :tNL, nil, EXPR_BEG)
1278
1242
  end
1279
1243
 
1280
1244
  def test_yylex_heredoc_none_squiggly
@@ -1282,37 +1246,41 @@ class TestRubyLexer < Minitest::Test
1282
1246
 
1283
1247
  assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
1284
1248
  nil,
1285
- :tIDENTIFIER, "a", :expr_cmdarg,
1286
- :tEQL, "=", :expr_beg,
1287
- :tSTRING_BEG, "\"", :expr_beg,
1288
- :tSTRING_CONTENT, "blah\nblah\n", :expr_beg,
1289
- :tSTRING_END, "EOF", :expr_end,
1290
- :tNL, nil, :expr_beg)
1249
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1250
+ :tEQL, "=", EXPR_BEG,
1251
+ :tSTRING_BEG, "\"", EXPR_BEG,
1252
+ :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1253
+ :tSTRING_END, "EOF", EXPR_LIT,
1254
+ :tNL, nil, EXPR_BEG)
1291
1255
  end
1292
1256
 
1293
1257
  def test_yylex_heredoc_single
1294
- assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1258
+ assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n\n\n42\n",
1295
1259
  nil,
1296
- :tIDENTIFIER, "a", :expr_cmdarg,
1297
- :tEQL, "=", :expr_beg,
1298
- :tSTRING_BEG, "\"", :expr_beg,
1299
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1300
- :tSTRING_END, "EOF", :expr_end,
1301
- :tNL, nil, :expr_beg)
1260
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1261
+ :tEQL, "=", EXPR_BEG,
1262
+ :tSTRING_BEG, "'", EXPR_BEG,
1263
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1264
+ :tSTRING_END, "EOF", EXPR_LIT,
1265
+ :tNL, nil, EXPR_BEG,
1266
+ :tINTEGER, 42, EXPR_LIT,
1267
+ :tNL, nil, EXPR_BEG)
1268
+
1269
+ assert_nil lex.old_ss
1302
1270
  end
1303
1271
 
1304
1272
  def test_yylex_heredoc_single_bad_eos_body
1305
1273
  refute_lex("a = <<'EOF'\nblah",
1306
1274
  :tIDENTIFIER, "a",
1307
1275
  :tEQL, "=",
1308
- :tSTRING_BEG, "\"")
1276
+ :tSTRING_BEG, "'")
1309
1277
  end
1310
1278
 
1311
1279
  def test_yylex_heredoc_single_bad_eos_empty
1312
1280
  refute_lex("a = <<''\n",
1313
1281
  :tIDENTIFIER, "a",
1314
1282
  :tEQL, "=",
1315
- :tSTRING_BEG, "\"")
1283
+ :tSTRING_BEG, "'")
1316
1284
  end
1317
1285
 
1318
1286
  def test_yylex_heredoc_single_bad_eos_term
@@ -1332,12 +1300,12 @@ class TestRubyLexer < Minitest::Test
1332
1300
  def test_yylex_heredoc_single_dash
1333
1301
  assert_lex3("a = <<-'EOF'\n blah blah\n EOF\n\n",
1334
1302
  nil,
1335
- :tIDENTIFIER, "a", :expr_cmdarg,
1336
- :tEQL, "=", :expr_beg,
1337
- :tSTRING_BEG, "\"", :expr_beg,
1338
- :tSTRING_CONTENT, " blah blah\n", :expr_beg,
1339
- :tSTRING_END, "EOF", :expr_end,
1340
- :tNL, nil, :expr_beg)
1303
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1304
+ :tEQL, "=", EXPR_BEG,
1305
+ :tSTRING_BEG, "'", EXPR_BEG,
1306
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1307
+ :tSTRING_END, "EOF", EXPR_LIT,
1308
+ :tNL, nil, EXPR_BEG)
1341
1309
  end
1342
1310
 
1343
1311
  def test_yylex_heredoc_single_squiggly
@@ -1345,96 +1313,84 @@ class TestRubyLexer < Minitest::Test
1345
1313
 
1346
1314
  assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
1347
1315
  nil,
1348
- :tIDENTIFIER, "a", :expr_cmdarg,
1349
- :tEQL, "=", :expr_beg,
1350
- :tSTRING_BEG, "\"", :expr_beg,
1351
- :tSTRING_CONTENT, "blah blah\n", :expr_beg,
1352
- :tSTRING_END, "EOF", :expr_end,
1353
- :tNL, nil, :expr_beg)
1316
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1317
+ :tEQL, "=", EXPR_BEG,
1318
+ :tSTRING_BEG, "'", EXPR_BEG,
1319
+ :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1320
+ :tSTRING_END, "EOF", EXPR_LIT,
1321
+ :tNL, nil, EXPR_BEG)
1354
1322
  end
1355
1323
 
1356
1324
  def test_yylex_identifier
1357
1325
  assert_lex3("identifier",
1358
1326
  nil,
1359
- :tIDENTIFIER, "identifier", :expr_cmdarg)
1327
+ :tIDENTIFIER, "identifier", EXPR_CMDARG)
1360
1328
  end
1361
1329
 
1362
1330
  def test_yylex_identifier_bang
1363
1331
  assert_lex3("identifier!",
1364
1332
  nil,
1365
- :tFID, "identifier!", :expr_cmdarg)
1333
+ :tFID, "identifier!", EXPR_CMDARG)
1366
1334
  end
1367
1335
 
1368
1336
  def test_yylex_identifier_cmp
1369
1337
  assert_lex_fname "<=>", :tCMP
1370
1338
  end
1371
1339
 
1372
- def test_yylex_identifier_def__18
1373
- setup_lexer_class RubyParser::V18
1374
-
1375
- assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
1376
- end
1377
-
1378
- def test_yylex_identifier_def__1920
1379
- setup_lexer_class RubyParser::V19
1340
+ def test_yylex_identifier_def__20
1341
+ setup_lexer_class RubyParser::V20
1380
1342
 
1381
- assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
1343
+ assert_lex_fname "identifier", :tIDENTIFIER, EXPR_ENDFN
1382
1344
  end
1383
1345
 
1384
1346
  def test_yylex_identifier_eh
1385
- assert_lex3("identifier?", nil, :tFID, "identifier?", :expr_cmdarg)
1386
- end
1387
-
1388
- def test_yylex_identifier_equals_arrow
1389
- assert_lex3(":blah==>",
1390
- nil,
1391
- :tSYMBOL, "blah=", :expr_end,
1392
- :tASSOC, "=>", :expr_beg)
1347
+ assert_lex3("identifier?", nil, :tFID, "identifier?", EXPR_CMDARG)
1393
1348
  end
1394
1349
 
1395
1350
  def test_yylex_identifier_equals3
1396
1351
  assert_lex3(":a===b",
1397
1352
  nil,
1398
- :tSYMBOL, "a", :expr_end,
1399
- :tEQQ, "===", :expr_beg,
1400
- :tIDENTIFIER, "b", :expr_arg)
1353
+ :tSYMBOL, "a", EXPR_LIT,
1354
+ :tEQQ, "===", EXPR_BEG,
1355
+ :tIDENTIFIER, "b", EXPR_ARG)
1401
1356
  end
1402
1357
 
1403
- def test_yylex_identifier_equals_equals_arrow
1404
- assert_lex3(":a==>b",
1358
+ def test_yylex_identifier_equals_arrow
1359
+ assert_lex3(":blah==>",
1405
1360
  nil,
1406
- :tSYMBOL, "a=", :expr_end,
1407
- :tASSOC, "=>", :expr_beg,
1408
- :tIDENTIFIER, "b", :expr_arg)
1361
+ :tSYMBOL, "blah=", EXPR_LIT,
1362
+ :tASSOC, "=>", EXPR_BEG)
1409
1363
  end
1410
1364
 
1411
1365
  def test_yylex_identifier_equals_caret
1412
1366
  assert_lex_fname "^", :tCARET
1413
1367
  end
1414
1368
 
1415
- def test_yylex_identifier_equals_def__18
1416
- setup_lexer_class RubyParser::V18
1417
-
1418
- assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
1369
+ def test_yylex_identifier_equals_def2
1370
+ assert_lex_fname "==", :tEQ
1419
1371
  end
1420
1372
 
1421
- def test_yylex_identifier_equals_def__1920
1422
- setup_lexer_class RubyParser::V19
1373
+ def test_yylex_identifier_equals_def__20
1374
+ setup_lexer_class RubyParser::V20
1423
1375
 
1424
- assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
1376
+ assert_lex_fname "identifier=", :tIDENTIFIER, EXPR_ENDFN
1425
1377
  end
1426
1378
 
1427
- def test_yylex_identifier_equals_def2
1428
- assert_lex_fname "==", :tEQ
1379
+ def test_yylex_identifier_equals_equals_arrow
1380
+ assert_lex3(":a==>b",
1381
+ nil,
1382
+ :tSYMBOL, "a=", EXPR_LIT,
1383
+ :tASSOC, "=>", EXPR_BEG,
1384
+ :tIDENTIFIER, "b", EXPR_ARG)
1429
1385
  end
1430
1386
 
1431
1387
  def test_yylex_identifier_equals_expr
1432
- self.lex_state = :expr_dot
1388
+ self.lex_state = EXPR_DOT
1433
1389
  assert_lex3("y = arg",
1434
1390
  nil,
1435
- :tIDENTIFIER, "y", :expr_cmdarg,
1436
- :tEQL, "=", :expr_beg,
1437
- :tIDENTIFIER, "arg", :expr_arg)
1391
+ :tIDENTIFIER, "y", EXPR_CMDARG,
1392
+ :tEQL, "=", EXPR_BEG,
1393
+ :tIDENTIFIER, "arg", EXPR_ARG)
1438
1394
  end
1439
1395
 
1440
1396
  def test_yylex_identifier_equals_or
@@ -1446,12 +1402,12 @@ class TestRubyLexer < Minitest::Test
1446
1402
  end
1447
1403
 
1448
1404
  def test_yylex_identifier_equals_tilde
1449
- self.lex_state = :expr_fname # can only set via parser's defs
1405
+ self.lex_state = EXPR_FNAME # can only set via parser's defs
1450
1406
 
1451
1407
  assert_lex3("identifier=~",
1452
1408
  nil,
1453
- :tIDENTIFIER, "identifier", :expr_endfn,
1454
- :tMATCH, "=~", :expr_beg)
1409
+ :tIDENTIFIER, "identifier", EXPR_ENDFN,
1410
+ :tMATCH, "=~", EXPR_BEG)
1455
1411
  end
1456
1412
 
1457
1413
  def test_yylex_identifier_gt
@@ -1479,11 +1435,11 @@ class TestRubyLexer < Minitest::Test
1479
1435
  end
1480
1436
 
1481
1437
  def test_yylex_integer
1482
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1438
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1483
1439
  end
1484
1440
 
1485
1441
  def test_yylex_integer_bin
1486
- assert_lex3("0b101010", nil, :tINTEGER, 42, :expr_end)
1442
+ assert_lex3("0b101010", nil, :tINTEGER, 42, EXPR_NUM)
1487
1443
  end
1488
1444
 
1489
1445
  def test_yylex_integer_bin_bad_none
@@ -1495,7 +1451,7 @@ class TestRubyLexer < Minitest::Test
1495
1451
  end
1496
1452
 
1497
1453
  def test_yylex_integer_dec
1498
- assert_lex3("42", nil, :tINTEGER, 42, :expr_end)
1454
+ assert_lex3("42", nil, :tINTEGER, 42, EXPR_NUM)
1499
1455
  end
1500
1456
 
1501
1457
  def test_yylex_integer_dec_bad_underscores
@@ -1503,7 +1459,7 @@ class TestRubyLexer < Minitest::Test
1503
1459
  end
1504
1460
 
1505
1461
  def test_yylex_integer_dec_d
1506
- assert_lex3("0d42", nil, :tINTEGER, 42, :expr_end)
1462
+ assert_lex3("0d42", nil, :tINTEGER, 42, EXPR_NUM)
1507
1463
  end
1508
1464
 
1509
1465
  def test_yylex_integer_dec_d_bad_none
@@ -1514,51 +1470,43 @@ class TestRubyLexer < Minitest::Test
1514
1470
  refute_lex "0d42__24"
1515
1471
  end
1516
1472
 
1517
- def test_yylex_integer_if_modifier
1518
- assert_lex3("123if",
1519
- nil,
1520
- :tINTEGER, 123, :expr_end,
1521
- :kIF_MOD, "if", :expr_beg)
1473
+ def test_yylex_integer_hex
1474
+ assert_lex3 "0x2a", nil, :tINTEGER, 42, EXPR_NUM
1522
1475
  end
1523
1476
 
1524
- def test_yylex_question_eh_a__18
1525
- setup_lexer_class RubyParser::V18
1526
-
1527
- assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
1477
+ def test_yylex_integer_hex_bad_none
1478
+ refute_lex "0x "
1528
1479
  end
1529
1480
 
1530
- def test_yylex_question_eh_a__19
1531
- setup_lexer_class RubyParser::V19
1532
-
1533
- assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
1481
+ def test_yylex_integer_hex_bad_underscores
1482
+ refute_lex "0xab__cd"
1534
1483
  end
1535
1484
 
1536
- def test_yylex_question_eh_escape_M_escape_C__18
1537
- setup_lexer_class RubyParser::V18
1538
-
1539
- assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
1485
+ def test_yylex_integer_if_modifier
1486
+ assert_lex3("123if",
1487
+ nil,
1488
+ :tINTEGER, 123, EXPR_NUM,
1489
+ :kIF_MOD, "if", EXPR_PAR)
1540
1490
  end
1541
1491
 
1542
- def test_yylex_question_eh_escape_M_escape_C__19
1543
- setup_lexer_class RubyParser::V19
1544
-
1545
- assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
1492
+ def test_yylex_integer_oct
1493
+ assert_lex3("052", nil, :tINTEGER, 42, EXPR_NUM)
1546
1494
  end
1547
1495
 
1548
- def test_yylex_integer_hex
1549
- assert_lex3 "0x2a", nil, :tINTEGER, 42, :expr_end
1496
+ def test_yylex_integer_oct_O
1497
+ assert_lex3 "0O52", nil, :tINTEGER, 42, EXPR_NUM
1550
1498
  end
1551
1499
 
1552
- def test_yylex_integer_hex_bad_none
1553
- refute_lex "0x "
1500
+ def test_yylex_integer_oct_O_bad_range
1501
+ refute_lex "0O8"
1554
1502
  end
1555
1503
 
1556
- def test_yylex_integer_hex_bad_underscores
1557
- refute_lex "0xab__cd"
1504
+ def test_yylex_integer_oct_O_bad_underscores
1505
+ refute_lex "0O1__23"
1558
1506
  end
1559
1507
 
1560
- def test_yylex_integer_oct
1561
- assert_lex3("052", nil, :tINTEGER, 42, :expr_end)
1508
+ def test_yylex_integer_oct_O_not_bad_none
1509
+ assert_lex3 "0O ", nil, :tINTEGER, 0, EXPR_NUM
1562
1510
  end
1563
1511
 
1564
1512
  def test_yylex_integer_oct_bad_range
@@ -1573,24 +1521,8 @@ class TestRubyLexer < Minitest::Test
1573
1521
  refute_lex "01__23"
1574
1522
  end
1575
1523
 
1576
- def test_yylex_integer_oct_O
1577
- assert_lex3 "0O52", nil, :tINTEGER, 42, :expr_end
1578
- end
1579
-
1580
- def test_yylex_integer_oct_O_bad_range
1581
- refute_lex "0O8"
1582
- end
1583
-
1584
- def test_yylex_integer_oct_O_bad_underscores
1585
- refute_lex "0O1__23"
1586
- end
1587
-
1588
- def test_yylex_integer_oct_O_not_bad_none
1589
- assert_lex3 "0O ", nil, :tINTEGER, 0, :expr_end
1590
- end
1591
-
1592
1524
  def test_yylex_integer_oct_o
1593
- assert_lex3 "0o52", nil, :tINTEGER, 42, :expr_end
1525
+ assert_lex3 "0o52", nil, :tINTEGER, 42, EXPR_NUM
1594
1526
  end
1595
1527
 
1596
1528
  def test_yylex_integer_oct_o_bad_range
@@ -1602,19 +1534,19 @@ class TestRubyLexer < Minitest::Test
1602
1534
  end
1603
1535
 
1604
1536
  def test_yylex_integer_oct_o_not_bad_none
1605
- assert_lex3 "0o ", nil, :tINTEGER, 0, :expr_end
1537
+ assert_lex3 "0o ", nil, :tINTEGER, 0, EXPR_NUM
1606
1538
  end
1607
1539
 
1608
1540
  def test_yylex_integer_trailing
1609
1541
  assert_lex3("1.to_s",
1610
1542
  nil,
1611
- :tINTEGER, 1, :expr_end,
1612
- :tDOT, ".", :expr_dot,
1613
- :tIDENTIFIER, "to_s", :expr_arg)
1543
+ :tINTEGER, 1, EXPR_NUM,
1544
+ :tDOT, ".", EXPR_DOT,
1545
+ :tIDENTIFIER, "to_s", EXPR_ARG)
1614
1546
  end
1615
1547
 
1616
1548
  def test_yylex_integer_underscore
1617
- assert_lex3("4_2", nil, :tINTEGER, 42, :expr_end)
1549
+ assert_lex3("4_2", nil, :tINTEGER, 42, EXPR_NUM)
1618
1550
  end
1619
1551
 
1620
1552
  def test_yylex_integer_underscore_bad
@@ -1622,11 +1554,64 @@ class TestRubyLexer < Minitest::Test
1622
1554
  end
1623
1555
 
1624
1556
  def test_yylex_integer_zero
1625
- assert_lex3 "0", nil, :tINTEGER, 0, :expr_end
1557
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1558
+ end
1559
+
1560
+ def test_yylex_is_your_spacebar_broken?
1561
+ assert_lex3(":a!=:b",
1562
+ nil,
1563
+ :tSYMBOL, "a", EXPR_LIT,
1564
+ :tNEQ, "!=", EXPR_BEG,
1565
+ :tSYMBOL, "b", EXPR_LIT)
1566
+ end
1567
+
1568
+ def test_yylex_iter_array_curly
1569
+ # this will lex, but doesn't parse... don't freak out.
1570
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1571
+ nil,
1572
+
1573
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1574
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1575
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1576
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1577
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1578
+ :tRBRACK, "]", EXPR_END, 0, 0,
1579
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1580
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1581
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1582
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1583
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1584
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1585
+ :tRCURLY, "}", EXPR_END, 0, 0)
1586
+ end
1587
+
1588
+ def test_yylex_iter_array_curly__24
1589
+ setup_lexer_class RubyParser::V24
1590
+
1591
+ assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1592
+ s(:iter,
1593
+ s(:call, nil, :f,
1594
+ s(:lit, :a).line(1),
1595
+ s(:array, s(:lit, :b).line(1)).line(1)).line(1),
1596
+ s(:args, :c, :d).line(1)).line(1),
1597
+
1598
+ :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1599
+ :tSYMBOL, "a", EXPR_LIT, 0, 0,
1600
+ :tCOMMA, ",", EXPR_PAR, 0, 0,
1601
+ :tLBRACK, "[", EXPR_PAR, 1, 0,
1602
+ :tSYMBOL, "b", EXPR_LIT, 1, 0,
1603
+ :tRBRACK, "]", EXPR_ENDARG, 0, 0,
1604
+ :tLBRACE_ARG, "{", EXPR_BEG, 0, 1,
1605
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1606
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 1,
1607
+ :tCOMMA, ",", EXPR_PAR, 0, 1,
1608
+ :tIDENTIFIER, "d", EXPR_ARG, 0, 1,
1609
+ :tPIPE, "|", EXPR_PAR, 0, 1,
1610
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1626
1611
  end
1627
1612
 
1628
1613
  def test_yylex_ivar
1629
- assert_lex3("@blah", nil, :tIVAR, "@blah", :expr_end)
1614
+ assert_lex3("@blah", nil, :tIVAR, "@blah", EXPR_END)
1630
1615
  end
1631
1616
 
1632
1617
  def test_yylex_ivar_bad
@@ -1634,338 +1619,609 @@ class TestRubyLexer < Minitest::Test
1634
1619
  end
1635
1620
 
1636
1621
  def test_yylex_ivar_bad_0_length
1637
- refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", :expr_end
1622
+ refute_lex "1+@\n", :tINTEGER, 1, :tPLUS, "+", EXPR_NUM
1638
1623
  end
1639
1624
 
1640
1625
  def test_yylex_keyword_expr
1641
- self.lex_state = :expr_endarg
1626
+ self.lex_state = EXPR_ENDARG
1627
+
1628
+ assert_lex3("if", nil, :kIF_MOD, "if", EXPR_PAR)
1629
+ end
1630
+
1631
+ def test_yylex_label
1632
+ assert_lex3("{a:",
1633
+ nil,
1634
+ :tLBRACE, "{", EXPR_PAR,
1635
+ :tLABEL, "a", EXPR_LAB)
1636
+ end
1637
+
1638
+ def test_yylex_label_in_params
1639
+ assert_lex3("foo(a:",
1640
+ nil,
1641
+ :tIDENTIFIER, "foo", EXPR_CMDARG,
1642
+ :tLPAREN2, "(", EXPR_PAR,
1643
+ :tLABEL, "a", EXPR_LAB)
1644
+ end
1645
+
1646
+ def test_yylex_lambda_args
1647
+ assert_lex("-> (a) { }",
1648
+ s(:iter, s(:lambda),
1649
+ s(:args, :a)),
1642
1650
 
1643
- assert_lex3("if", nil, :kIF_MOD, "if", :expr_beg)
1651
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1652
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1653
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1654
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1655
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1656
+ :tRCURLY, "}", EXPR_END, 0, 0)
1657
+ end
1658
+
1659
+ def test_yylex_lambda_args__24
1660
+ setup_lexer_class RubyParser::V24
1661
+
1662
+ assert_lex("-> (a) { }",
1663
+ s(:iter, s(:lambda),
1664
+ s(:args, :a)),
1665
+
1666
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1667
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1668
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1669
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1670
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1671
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1672
+ end
1673
+
1674
+ def test_yylex_lambda_args_opt
1675
+ assert_lex("-> (a=nil) { }",
1676
+ s(:iter, s(:lambda),
1677
+ s(:args, s(:lasgn, :a, s(:nil)))),
1678
+
1679
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1680
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1681
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1682
+ :tEQL, "=", EXPR_BEG, 1, 0,
1683
+ :kNIL, "nil", EXPR_END, 1, 0,
1684
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1685
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1686
+ :tRCURLY, "}", EXPR_END, 0, 0)
1687
+ end
1688
+
1689
+ def test_yylex_lambda_args_opt__24
1690
+ setup_lexer_class RubyParser::V24
1691
+
1692
+ assert_lex("-> (a=nil) { }",
1693
+ s(:iter, s(:lambda),
1694
+ s(:args, s(:lasgn, :a, s(:nil)))),
1695
+
1696
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1697
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1698
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1699
+ :tEQL, "=", EXPR_BEG, 1, 0,
1700
+ :kNIL, "nil", EXPR_END, 1, 0,
1701
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1702
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1703
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1704
+ end
1705
+
1706
+ def test_yylex_lambda_as_args_with_block
1707
+ assert_lex3("a -> do end do end",
1708
+ nil,
1709
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1710
+ :tLAMBDA, "->", EXPR_ENDFN,
1711
+ :kDO, "do", EXPR_BEG,
1712
+ :kEND, "end", EXPR_END,
1713
+ :kDO, "do", EXPR_BEG,
1714
+ :kEND, "end", EXPR_END)
1715
+ end
1716
+
1717
+ def test_yylex_lambda_hash
1718
+ assert_lex("-> (a={}) { }",
1719
+ s(:iter, s(:lambda),
1720
+ s(:args, s(:lasgn, :a, s(:hash)))),
1721
+
1722
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1723
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1724
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1725
+ :tEQL, "=", EXPR_BEG, 1, 0,
1726
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1727
+ :tRCURLY, "}", EXPR_END, 1, 0,
1728
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1729
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1730
+ :tRCURLY, "}", EXPR_END, 0, 0)
1731
+ end
1732
+
1733
+ def test_yylex_lambda_hash__24
1734
+ setup_lexer_class RubyParser::V24
1735
+
1736
+ assert_lex("-> (a={}) { }",
1737
+ s(:iter, s(:lambda),
1738
+ s(:args, s(:lasgn, :a, s(:hash)))),
1739
+
1740
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1741
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1742
+ :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1743
+ :tEQL, "=", EXPR_BEG, 1, 0,
1744
+ :tLBRACE, "{", EXPR_PAR, 1, 1,
1745
+ :tRCURLY, "}", EXPR_ENDARG, 1, 0,
1746
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1747
+ :tLCURLY, "{", EXPR_PAR, 0, 1,
1748
+ :tRCURLY, "}", EXPR_ENDARG, 0, 0)
1749
+ end
1750
+
1751
+ def test_yylex_lasgn_call_same_name
1752
+ assert_lex("a = b.c :d => 1",
1753
+ s(:lasgn, :a,
1754
+ s(:call, s(:call, nil, :b), :c,
1755
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1756
+
1757
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1758
+ :tEQL, "=", EXPR_BEG, 0, 0,
1759
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1760
+ :tDOT, ".", EXPR_DOT, 0, 0,
1761
+ :tIDENTIFIER, "c", EXPR_ARG, 0, 0, # different
1762
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1763
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1764
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1765
+
1766
+ assert_lex("a = b.a :d => 1",
1767
+ s(:lasgn, :a,
1768
+ s(:call, s(:call, nil, :b), :a,
1769
+ s(:hash, s(:lit, :d), s(:lit, 1)))),
1770
+
1771
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1772
+ :tEQL, "=", EXPR_BEG, 0, 0,
1773
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1774
+ :tDOT, ".", EXPR_DOT, 0, 0,
1775
+ :tIDENTIFIER, "a", EXPR_ARG, 0, 0, # same as lvar
1776
+ :tSYMBOL, "d", EXPR_LIT, 0, 0,
1777
+ :tASSOC, "=>", EXPR_BEG, 0, 0,
1778
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1644
1779
  end
1645
1780
 
1646
1781
  def test_yylex_lt
1647
- assert_lex3("<", nil, :tLT, "<", :expr_beg)
1782
+ assert_lex3("<", nil, :tLT, "<", EXPR_BEG)
1648
1783
  end
1649
1784
 
1650
1785
  def test_yylex_lt2
1651
1786
  assert_lex3("a << b",
1652
1787
  nil,
1653
- :tIDENTIFIER, "a", :expr_cmdarg,
1654
- :tLSHFT, "<<", :expr_beg,
1655
- :tIDENTIFIER, "b", :expr_arg)
1788
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1789
+ :tLSHFT, "<<", EXPR_BEG,
1790
+ :tIDENTIFIER, "b", EXPR_ARG)
1656
1791
  end
1657
1792
 
1658
1793
  def test_yylex_lt2_equals
1659
1794
  assert_lex3("a <<= b",
1660
1795
  nil,
1661
- :tIDENTIFIER, "a", :expr_cmdarg,
1662
- :tOP_ASGN, "<<", :expr_beg,
1663
- :tIDENTIFIER, "b", :expr_arg)
1796
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1797
+ :tOP_ASGN, "<<", EXPR_BEG,
1798
+ :tIDENTIFIER, "b", EXPR_ARG)
1664
1799
  end
1665
1800
 
1666
1801
  def test_yylex_lt_equals
1667
- assert_lex3("<=", nil, :tLEQ, "<=", :expr_beg)
1802
+ assert_lex3("<=", nil, :tLEQ, "<=", EXPR_BEG)
1803
+ end
1804
+
1805
+ def test_yylex_method_parens_chevron
1806
+ assert_lex("a()<<1",
1807
+ s(:call, s(:call, nil, :a), :<<, s(:lit, 1)),
1808
+ :tIDENTIFIER, "a", EXPR_CMDARG, 0, 0,
1809
+ :tLPAREN2, "(", EXPR_PAR, 1, 0,
1810
+ :tRPAREN, ")", EXPR_ENDFN, 0, 0,
1811
+ :tLSHFT, "<<", EXPR_BEG, 0, 0,
1812
+ :tINTEGER, 1, EXPR_NUM, 0, 0)
1668
1813
  end
1669
1814
 
1670
1815
  def test_yylex_minus
1671
1816
  assert_lex3("1 - 2",
1672
1817
  nil,
1673
- :tINTEGER, 1, :expr_end,
1674
- :tMINUS, "-", :expr_beg,
1675
- :tINTEGER, 2, :expr_end)
1818
+ :tINTEGER, 1, EXPR_NUM,
1819
+ :tMINUS, "-", EXPR_BEG,
1820
+ :tINTEGER, 2, EXPR_NUM)
1676
1821
  end
1677
1822
 
1678
1823
  def test_yylex_minus_equals
1679
- assert_lex3("-=", nil, :tOP_ASGN, "-", :expr_beg)
1824
+ assert_lex3("-=", nil, :tOP_ASGN, "-", EXPR_BEG)
1680
1825
  end
1681
1826
 
1682
1827
  def test_yylex_minus_method
1683
- self.lex_state = :expr_fname
1828
+ self.lex_state = EXPR_FNAME
1684
1829
 
1685
- assert_lex3("-", nil, :tMINUS, "-", :expr_arg)
1830
+ assert_lex3("-", nil, :tMINUS, "-", EXPR_ARG)
1686
1831
  end
1687
1832
 
1688
1833
  def test_yylex_minus_unary_method
1689
- self.lex_state = :expr_fname
1834
+ self.lex_state = EXPR_FNAME
1690
1835
 
1691
- assert_lex3("-@", nil, :tUMINUS, "-@", :expr_arg)
1836
+ assert_lex3("-@", nil, :tUMINUS, "-@", EXPR_ARG)
1692
1837
  end
1693
1838
 
1694
1839
  def test_yylex_minus_unary_number
1695
1840
  assert_lex3("-42",
1696
1841
  nil,
1697
- :tUMINUS_NUM, "-", :expr_beg,
1698
- :tINTEGER, 42, :expr_end)
1842
+ :tUMINUS_NUM, "-", EXPR_BEG,
1843
+ :tINTEGER, 42, EXPR_NUM)
1844
+ end
1845
+
1846
+ def test_yylex_not_at_defn
1847
+ assert_lex("def +@; end",
1848
+ s(:defn, :+@, s(:args), s(:nil)),
1849
+
1850
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1851
+ :tUPLUS, "+@", EXPR_ARG, 0, 0,
1852
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1853
+ :kEND, "end", EXPR_END, 0, 0)
1854
+
1855
+ assert_lex("def !@; end",
1856
+ s(:defn, :"!@", s(:args), s(:nil)),
1857
+
1858
+ :kDEF, "def", EXPR_FNAME, 0, 0,
1859
+ :tBANG, "!@", EXPR_ARG, 0, 0,
1860
+ :tSEMI, ";", EXPR_BEG, 0, 0,
1861
+ :kEND, "end", EXPR_END, 0, 0)
1862
+ end
1863
+
1864
+ def test_yylex_not_at_ivar
1865
+ assert_lex("!@ivar",
1866
+ s(:call, s(:ivar, :@ivar).line(1), :"!").line(1),
1867
+
1868
+ :tBANG, "!", EXPR_BEG, 0, 0,
1869
+ :tIVAR, "@ivar", EXPR_END, 0, 0)
1870
+ end
1871
+
1872
+ def test_yylex_not_unary_method
1873
+ self.lex_state = EXPR_FNAME
1874
+
1875
+ assert_lex3("!@", nil, :tBANG, "!@", EXPR_ARG)
1699
1876
  end
1700
1877
 
1701
1878
  def test_yylex_nth_ref
1702
1879
  assert_lex3("[$1, $2, $3, $4, $5, $6, $7, $8, $9]",
1703
1880
  nil,
1704
- :tLBRACK, "[", :expr_beg,
1705
- :tNTH_REF, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1706
- :tNTH_REF, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1707
- :tNTH_REF, 3, :expr_end, :tCOMMA, ",", :expr_beg,
1708
- :tNTH_REF, 4, :expr_end, :tCOMMA, ",", :expr_beg,
1709
- :tNTH_REF, 5, :expr_end, :tCOMMA, ",", :expr_beg,
1710
- :tNTH_REF, 6, :expr_end, :tCOMMA, ",", :expr_beg,
1711
- :tNTH_REF, 7, :expr_end, :tCOMMA, ",", :expr_beg,
1712
- :tNTH_REF, 8, :expr_end, :tCOMMA, ",", :expr_beg,
1713
- :tNTH_REF, 9, :expr_end,
1714
- :tRBRACK, "]", :expr_endarg)
1881
+ :tLBRACK, "[", EXPR_PAR,
1882
+ :tNTH_REF, 1, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1883
+ :tNTH_REF, 2, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1884
+ :tNTH_REF, 3, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1885
+ :tNTH_REF, 4, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1886
+ :tNTH_REF, 5, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1887
+ :tNTH_REF, 6, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1888
+ :tNTH_REF, 7, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1889
+ :tNTH_REF, 8, EXPR_END, :tCOMMA, ",", EXPR_PAR,
1890
+ :tNTH_REF, 9, EXPR_END,
1891
+ :tRBRACK, "]", EXPR_END)
1715
1892
  end
1716
1893
 
1717
- def test_yylex_open_bracket
1718
- assert_lex3("(", nil, :tLPAREN, "(", :expr_beg)
1894
+ def test_yylex_number_times_ident_times_return_number
1895
+ assert_lex("1 * b * 3",
1896
+ s(:call,
1897
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1898
+ :*, s(:lit, 3)),
1899
+
1900
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1901
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1902
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1903
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1904
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1905
+
1906
+ assert_lex("1 * b *\n 3",
1907
+ s(:call,
1908
+ s(:call, s(:lit, 1), :*, s(:call, nil, :b)),
1909
+ :*, s(:lit, 3)),
1910
+
1911
+ :tINTEGER, 1, EXPR_NUM, 0, 0,
1912
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1913
+ :tIDENTIFIER, "b", EXPR_ARG, 0, 0,
1914
+ :tSTAR2, "*", EXPR_BEG, 0, 0,
1915
+ :tINTEGER, 3, EXPR_NUM, 0, 0)
1719
1916
  end
1720
1917
 
1721
- def test_yylex_open_bracket_cmdarg
1722
- self.lex_state = :expr_cmdarg
1918
+ def test_yylex_numbers
1919
+ assert_lex3 "0b10", nil, :tINTEGER, 2, EXPR_NUM
1920
+ assert_lex3 "0B10", nil, :tINTEGER, 2, EXPR_NUM
1921
+
1922
+ assert_lex3 "0d10", nil, :tINTEGER, 10, EXPR_NUM
1923
+ assert_lex3 "0D10", nil, :tINTEGER, 10, EXPR_NUM
1924
+
1925
+ assert_lex3 "0x10", nil, :tINTEGER, 16, EXPR_NUM
1926
+ assert_lex3 "0X10", nil, :tINTEGER, 16, EXPR_NUM
1927
+
1928
+ assert_lex3 "0o10", nil, :tINTEGER, 8, EXPR_NUM
1929
+ assert_lex3 "0O10", nil, :tINTEGER, 8, EXPR_NUM
1930
+
1931
+ assert_lex3 "0o", nil, :tINTEGER, 0, EXPR_NUM
1932
+ assert_lex3 "0O", nil, :tINTEGER, 0, EXPR_NUM
1933
+
1934
+ assert_lex3 "0", nil, :tINTEGER, 0, EXPR_NUM
1935
+
1936
+ refute_lex "0x"
1937
+ refute_lex "0X"
1938
+ refute_lex "0b"
1939
+ refute_lex "0B"
1940
+ refute_lex "0d"
1941
+ refute_lex "0D"
1942
+
1943
+ refute_lex "08"
1944
+ refute_lex "09"
1945
+ refute_lex "0o8"
1946
+ refute_lex "0o9"
1947
+ refute_lex "0O8"
1948
+ refute_lex "0O9"
1723
1949
 
1724
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1950
+ refute_lex "1_e1"
1951
+ refute_lex "1_.1"
1952
+ refute_lex "1__1"
1725
1953
  end
1726
1954
 
1727
- def test_yylex_open_bracket_exprarg__18
1728
- setup_lexer_class RubyParser::V18
1729
- self.lex_state = :expr_arg
1955
+ def test_yylex_open_bracket
1956
+ assert_lex3("(", nil, :tLPAREN, "(", EXPR_PAR)
1957
+ end
1730
1958
 
1731
- assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
1959
+ def test_yylex_open_bracket_cmdarg
1960
+ self.lex_state = EXPR_CMDARG
1961
+
1962
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1732
1963
  end
1733
1964
 
1734
- def test_yylex_open_bracket_exprarg__19
1735
- setup_lexer_class RubyParser::V19
1736
- self.lex_state = :expr_arg
1965
+ def test_yylex_open_bracket_exprarg__20
1966
+ setup_lexer_class RubyParser::V20
1967
+ self.lex_state = EXPR_ARG
1737
1968
 
1738
- assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
1969
+ assert_lex3(" (", nil, :tLPAREN_ARG, "(", EXPR_PAR)
1739
1970
  end
1740
1971
 
1741
1972
  def test_yylex_open_curly_bracket
1742
- assert_lex3("{", nil, :tLBRACE, "{", :expr_beg)
1973
+ assert_lex3("{", nil, :tLBRACE, "{", EXPR_PAR)
1743
1974
  end
1744
1975
 
1745
1976
  def test_yylex_open_curly_bracket_arg
1746
- self.lex_state = :expr_arg
1977
+ self.lex_state = EXPR_ARG
1747
1978
 
1748
1979
  assert_lex3("m { 3 }",
1749
1980
  nil,
1750
- :tIDENTIFIER, "m", :expr_cmdarg,
1751
- :tLCURLY, "{", :expr_beg,
1752
- :tINTEGER, 3, :expr_end,
1753
- :tRCURLY, "}", :expr_endarg)
1981
+ :tIDENTIFIER, "m", EXPR_CMDARG,
1982
+ :tLCURLY, "{", EXPR_PAR,
1983
+ :tINTEGER, 3, EXPR_NUM,
1984
+ :tRCURLY, "}", EXPR_END)
1754
1985
  end
1755
1986
 
1756
1987
  def test_yylex_open_curly_bracket_block
1757
- self.lex_state = :expr_endarg # seen m(3)
1988
+ self.lex_state = EXPR_ENDARG # seen m(3)
1758
1989
 
1759
1990
  assert_lex3("{ 4 }",
1760
1991
  nil,
1761
- :tLBRACE_ARG, "{", :expr_beg,
1762
- :tINTEGER, 4, :expr_end,
1763
- :tRCURLY, "}", :expr_endarg)
1992
+ :tLBRACE_ARG, "{", EXPR_BEG,
1993
+ :tINTEGER, 4, EXPR_NUM,
1994
+ :tRCURLY, "}", EXPR_END)
1764
1995
  end
1765
1996
 
1766
1997
  def test_yylex_open_square_bracket_arg
1767
- self.lex_state = :expr_arg
1998
+ self.lex_state = EXPR_ARG
1768
1999
 
1769
2000
  assert_lex3("m [ 3 ]",
1770
2001
  nil,
1771
- :tIDENTIFIER, "m", :expr_cmdarg,
1772
- :tLBRACK, "[", :expr_beg,
1773
- :tINTEGER, 3, :expr_end,
1774
- :tRBRACK, "]", :expr_endarg)
2002
+ :tIDENTIFIER, "m", EXPR_CMDARG,
2003
+ :tLBRACK, "[", EXPR_PAR,
2004
+ :tINTEGER, 3, EXPR_NUM,
2005
+ :tRBRACK, "]", EXPR_END)
1775
2006
  end
1776
2007
 
1777
2008
  def test_yylex_open_square_bracket_ary
1778
2009
  assert_lex3("[1, 2, 3]",
1779
2010
  nil,
1780
- :tLBRACK, "[", :expr_beg,
1781
- :tINTEGER, 1, :expr_end, :tCOMMA, ",", :expr_beg,
1782
- :tINTEGER, 2, :expr_end, :tCOMMA, ",", :expr_beg,
1783
- :tINTEGER, 3, :expr_end,
1784
- :tRBRACK, "]", :expr_endarg)
2011
+ :tLBRACK, "[", EXPR_PAR,
2012
+ :tINTEGER, 1, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
2013
+ :tINTEGER, 2, EXPR_NUM, :tCOMMA, ",", EXPR_PAR,
2014
+ :tINTEGER, 3, EXPR_NUM,
2015
+ :tRBRACK, "]", EXPR_END)
1785
2016
  end
1786
2017
 
1787
2018
  def test_yylex_open_square_bracket_meth
1788
2019
  assert_lex3("m[3]",
1789
2020
  nil,
1790
- :tIDENTIFIER, "m", :expr_cmdarg,
1791
- :tLBRACK2, "[", :expr_beg,
1792
- :tINTEGER, 3, :expr_end,
1793
- :tRBRACK, "]", :expr_endarg)
2021
+ :tIDENTIFIER, "m", EXPR_CMDARG,
2022
+ :tLBRACK2, "[", EXPR_PAR,
2023
+ :tINTEGER, 3, EXPR_NUM,
2024
+ :tRBRACK, "]", EXPR_END)
1794
2025
  end
1795
2026
 
1796
2027
  def test_yylex_or
1797
- assert_lex3("|", nil, :tPIPE, "|", :expr_beg)
2028
+ assert_lex3("|", nil, :tPIPE, "|", EXPR_PAR)
1798
2029
  end
1799
2030
 
1800
2031
  def test_yylex_or2
1801
- assert_lex3("||", nil, :tOROP, "||", :expr_beg)
2032
+ assert_lex3("||", nil, :tOROP, "||", EXPR_BEG)
1802
2033
  end
1803
2034
 
1804
2035
  def test_yylex_or2_equals
1805
- assert_lex3("||=", nil, :tOP_ASGN, "||", :expr_beg)
2036
+ assert_lex3("||=", nil, :tOP_ASGN, "||", EXPR_BEG)
1806
2037
  end
1807
2038
 
1808
2039
  def test_yylex_or_equals
1809
- assert_lex3("|=", nil, :tOP_ASGN, "|", :expr_beg)
2040
+ assert_lex3("|=", nil, :tOP_ASGN, "|", EXPR_BEG)
2041
+ end
2042
+
2043
+ def test_yylex_paren_string_interpolated_regexp
2044
+ setup_lexer('%( #{(/abcd/)} )',
2045
+ s(:dstr, " ", s(:evstr, s(:lit, /abcd/)), s(:str, " ")))
2046
+
2047
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2048
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
2049
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2050
+
2051
+ emulate_string_interpolation do
2052
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
2053
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
2054
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
2055
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
2056
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
2057
+ end
2058
+
2059
+ assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
2060
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
2061
+
2062
+ refute_lexeme
2063
+ end
2064
+
2065
+ def test_yylex_paren_string_parens_interpolated
2066
+ setup_lexer('%((#{b}#{d}))',
2067
+ s(:dstr,
2068
+ "(",
2069
+ s(:evstr, s(:call, nil, :b)),
2070
+ s(:evstr, s(:call, nil, :d)),
2071
+ s(:str, ")")))
2072
+
2073
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2074
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2075
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2076
+
2077
+ emulate_string_interpolation do
2078
+ assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2079
+ end
2080
+
2081
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2082
+
2083
+ emulate_string_interpolation do
2084
+ assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
2085
+ end
2086
+
2087
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2088
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
2089
+
2090
+ refute_lexeme
2091
+ end
2092
+
2093
+ def test_yylex_paren_string_parens_interpolated_regexp
2094
+ setup_lexer('%((#{(/abcd/)}))',
2095
+ s(:dstr, "(", s(:evstr, s(:lit, /abcd/)), s(:str, ")")))
2096
+
2097
+ assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2098
+ assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2099
+
2100
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2101
+
2102
+ emulate_string_interpolation do
2103
+ assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
2104
+ assert_next_lexeme :tREGEXP_BEG, "/", EXPR_PAR, 1, 0
2105
+ assert_next_lexeme :tSTRING_CONTENT, "abcd", EXPR_PAR, 1, 0
2106
+ assert_next_lexeme :tREGEXP_END, "", EXPR_LIT, 1, 0
2107
+ assert_next_lexeme :tRPAREN, ")", EXPR_ENDFN, 0, 0
2108
+ end
2109
+
2110
+ assert_next_lexeme :tSTRING_CONTENT, ")", EXPR_BEG, 0, 0
2111
+ assert_next_lexeme :tSTRING_END, ")", EXPR_LIT, 0, 0
2112
+
2113
+ refute_lexeme
1810
2114
  end
1811
2115
 
1812
2116
  def test_yylex_percent
1813
2117
  assert_lex3("a % 2",
1814
2118
  nil,
1815
- :tIDENTIFIER, "a", :expr_cmdarg,
1816
- :tPERCENT, "%", :expr_beg,
1817
- :tINTEGER, 2, :expr_end)
2119
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2120
+ :tPERCENT, "%", EXPR_BEG,
2121
+ :tINTEGER, 2, EXPR_NUM)
1818
2122
  end
1819
2123
 
1820
2124
  def test_yylex_percent_equals
1821
2125
  assert_lex3("a %= 2",
1822
2126
  nil,
1823
- :tIDENTIFIER, "a", :expr_cmdarg,
1824
- :tOP_ASGN, "%", :expr_beg,
1825
- :tINTEGER, 2, :expr_end)
2127
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2128
+ :tOP_ASGN, "%", EXPR_BEG,
2129
+ :tINTEGER, 2, EXPR_NUM)
1826
2130
  end
1827
2131
 
1828
2132
  def test_yylex_plus
1829
2133
  assert_lex3("1 + 1", # TODO lex_state?
1830
2134
  nil,
1831
- :tINTEGER, 1, :expr_end,
1832
- :tPLUS, "+", :expr_beg,
1833
- :tINTEGER, 1, :expr_end)
2135
+ :tINTEGER, 1, EXPR_NUM,
2136
+ :tPLUS, "+", EXPR_BEG,
2137
+ :tINTEGER, 1, EXPR_NUM)
1834
2138
  end
1835
2139
 
1836
2140
  def test_yylex_plus_equals
1837
- assert_lex3("+=", nil, :tOP_ASGN, "+", :expr_beg)
2141
+ assert_lex3("+=", nil, :tOP_ASGN, "+", EXPR_BEG)
1838
2142
  end
1839
2143
 
1840
2144
  def test_yylex_plus_method
1841
- self.lex_state = :expr_fname
2145
+ self.lex_state = EXPR_FNAME
1842
2146
 
1843
- assert_lex3("+", nil, :tPLUS, "+", :expr_arg)
2147
+ assert_lex3("+", nil, :tPLUS, "+", EXPR_ARG)
1844
2148
  end
1845
2149
 
1846
2150
  def test_yylex_plus_unary_method
1847
- self.lex_state = :expr_fname
1848
-
1849
- assert_lex3("+@", nil, :tUPLUS, "+@", :expr_arg)
1850
- end
1851
-
1852
- def test_yylex_not_unary_method
1853
- self.lex_state = :expr_fname
2151
+ self.lex_state = EXPR_FNAME
1854
2152
 
1855
- assert_lex3("!@", nil, :tUBANG, "!@", :expr_arg)
2153
+ assert_lex3("+@", nil, :tUPLUS, "+@", EXPR_ARG)
1856
2154
  end
1857
2155
 
1858
- def test_yylex_numbers
1859
- assert_lex3("0b10", nil, :tINTEGER, 2, :expr_end)
1860
- assert_lex3("0B10", nil, :tINTEGER, 2, :expr_end)
1861
-
1862
- assert_lex3("0d10", nil, :tINTEGER, 10, :expr_end)
1863
- assert_lex3("0D10", nil, :tINTEGER, 10, :expr_end)
1864
-
1865
- assert_lex3("0x10", nil, :tINTEGER, 16, :expr_end)
1866
- assert_lex3("0X10", nil, :tINTEGER, 16, :expr_end)
1867
-
1868
- assert_lex3("0o10", nil, :tINTEGER, 8, :expr_end)
1869
- assert_lex3("0O10", nil, :tINTEGER, 8, :expr_end)
1870
-
1871
- assert_lex3("0o", nil, :tINTEGER, 0, :expr_end)
1872
- assert_lex3("0O", nil, :tINTEGER, 0, :expr_end)
1873
-
1874
- assert_lex3("0", nil, :tINTEGER, 0, :expr_end)
1875
-
1876
- refute_lex "0x"
1877
- refute_lex "0X"
1878
- refute_lex "0b"
1879
- refute_lex "0B"
1880
- refute_lex "0d"
1881
- refute_lex "0D"
1882
-
1883
- refute_lex "08"
1884
- refute_lex "09"
1885
- refute_lex "0o8"
1886
- refute_lex "0o9"
1887
- refute_lex "0O8"
1888
- refute_lex "0O9"
1889
-
1890
- refute_lex "1_e1"
1891
- refute_lex "1_.1"
1892
- refute_lex "1__1"
2156
+ def test_yylex_plus_unary_number
2157
+ assert_lex3("+42", nil, :tINTEGER, 42, EXPR_NUM)
1893
2158
  end
1894
2159
 
1895
- def test_yylex_plus_unary_number
1896
- assert_lex3("+42", nil, :tINTEGER, 42, :expr_end)
2160
+ def test_yylex_question_bad_eos
2161
+ refute_lex "?"
1897
2162
  end
1898
2163
 
1899
- def test_yylex_question__18
1900
- setup_lexer_class RubyParser::V18
2164
+ def test_yylex_question_eh_a__20
2165
+ setup_lexer_class RubyParser::V20
1901
2166
 
1902
- assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
2167
+ assert_lex3("?a", nil, :tSTRING, "a", EXPR_END)
1903
2168
  end
1904
2169
 
1905
- def test_yylex_question__19
1906
- setup_lexer_class RubyParser::V19
2170
+ def test_yylex_question_eh_escape_M_escape_C__20
2171
+ setup_lexer_class RubyParser::V20
1907
2172
 
1908
- assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
2173
+ assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", EXPR_END)
1909
2174
  end
1910
2175
 
1911
- def test_yylex_question_bad_eos
1912
- refute_lex "?"
2176
+ def test_yylex_question_control_escape
2177
+ assert_lex3('?\C-\]', nil, :tSTRING, ?\C-\], EXPR_END)
1913
2178
  end
1914
2179
 
1915
2180
  def test_yylex_question_ws
1916
- assert_lex3("? ", nil, :tEH, "?", :expr_value)
1917
- assert_lex3("?\n", nil, :tEH, "?", :expr_value)
1918
- assert_lex3("?\t", nil, :tEH, "?", :expr_value)
1919
- assert_lex3("?\v", nil, :tEH, "?", :expr_value)
1920
- assert_lex3("?\r", nil, :tEH, "?", :expr_value)
1921
- assert_lex3("?\f", nil, :tEH, "?", :expr_value)
1922
- end
1923
-
1924
- def test_yylex_question_ws_backslashed__18
1925
- setup_lexer_class RubyParser::V18
1926
-
1927
- assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
1928
- assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
1929
- assert_lex3("?\\t", nil, :tINTEGER, 9, :expr_end)
1930
- assert_lex3("?\\v", nil, :tINTEGER, 11, :expr_end)
1931
- assert_lex3("?\\r", nil, :tINTEGER, 13, :expr_end)
1932
- assert_lex3("?\\f", nil, :tINTEGER, 12, :expr_end)
2181
+ assert_lex3("? ", nil, :tEH, "?", EXPR_BEG)
2182
+ assert_lex3("?\n", nil, :tEH, "?", EXPR_BEG)
2183
+ assert_lex3("?\t", nil, :tEH, "?", EXPR_BEG)
2184
+ assert_lex3("?\v", nil, :tEH, "?", EXPR_BEG)
2185
+ assert_lex3("?\r", nil, :tEH, "?", EXPR_BEG)
2186
+ assert_lex3("?\f", nil, :tEH, "?", EXPR_BEG)
1933
2187
  end
1934
2188
 
1935
- def test_yylex_question_ws_backslashed__19
1936
- setup_lexer_class RubyParser::V19
2189
+ def test_yylex_question_ws_backslashed__20
2190
+ setup_lexer_class RubyParser::V20
1937
2191
 
1938
- assert_lex3("?\\ ", nil, :tSTRING, " ", :expr_end)
1939
- assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
1940
- assert_lex3("?\\t", nil, :tSTRING, "\t", :expr_end)
1941
- assert_lex3("?\\v", nil, :tSTRING, "\v", :expr_end)
1942
- assert_lex3("?\\r", nil, :tSTRING, "\r", :expr_end)
1943
- assert_lex3("?\\f", nil, :tSTRING, "\f", :expr_end)
2192
+ assert_lex3("?\\ ", nil, :tSTRING, " ", EXPR_END)
2193
+ assert_lex3("?\\n", nil, :tSTRING, "\n", EXPR_END)
2194
+ assert_lex3("?\\t", nil, :tSTRING, "\t", EXPR_END)
2195
+ assert_lex3("?\\v", nil, :tSTRING, "\v", EXPR_END)
2196
+ assert_lex3("?\\r", nil, :tSTRING, "\r", EXPR_END)
2197
+ assert_lex3("?\\f", nil, :tSTRING, "\f", EXPR_END)
1944
2198
  end
1945
2199
 
1946
2200
  def test_yylex_rbracket
1947
- assert_lex3("]", nil, :tRBRACK, "]", :expr_endarg)
2201
+ assert_lex3("]", nil, :tRBRACK, "]", EXPR_END)
1948
2202
  end
1949
2203
 
1950
2204
  def test_yylex_rcurly
1951
- assert_lex3("}", nil, :tRCURLY, "}", :expr_endarg)
2205
+ assert_lex("}", nil, :tRCURLY, "}", EXPR_END, 0, 1) do
2206
+ lexer.brace_nest += 2
2207
+ end
1952
2208
  end
1953
2209
 
1954
2210
  def test_yylex_regexp
1955
2211
  assert_lex3("/regexp/",
1956
2212
  nil,
1957
- :tREGEXP_BEG, "/", :expr_beg,
1958
- :tSTRING_CONTENT, "regexp", :expr_beg,
1959
- :tREGEXP_END, "", :expr_end)
2213
+ :tREGEXP_BEG, "/", EXPR_BEG,
2214
+ :tSTRING_CONTENT, "regexp", EXPR_BEG,
2215
+ :tREGEXP_END, "", EXPR_LIT)
1960
2216
  end
1961
2217
 
1962
2218
  def test_yylex_regexp_ambiguous
1963
2219
  assert_lex3("method /regexp/",
1964
2220
  nil,
1965
- :tIDENTIFIER, "method", :expr_cmdarg,
1966
- :tREGEXP_BEG, "/", :expr_cmdarg,
1967
- :tSTRING_CONTENT, "regexp", :expr_cmdarg,
1968
- :tREGEXP_END, "", :expr_end)
2221
+ :tIDENTIFIER, "method", EXPR_CMDARG,
2222
+ :tREGEXP_BEG, "/", EXPR_CMDARG,
2223
+ :tSTRING_CONTENT, "regexp", EXPR_CMDARG,
2224
+ :tREGEXP_END, "", EXPR_LIT)
1969
2225
  end
1970
2226
 
1971
2227
  def test_yylex_regexp_bad
@@ -1977,25 +2233,26 @@ class TestRubyLexer < Minitest::Test
1977
2233
  def test_yylex_regexp_escape_C
1978
2234
  assert_lex3("/regex\\C-x/",
1979
2235
  nil,
1980
- :tREGEXP_BEG, "/", :expr_beg,
1981
- :tSTRING_CONTENT, "regex\\C-x", :expr_beg,
1982
- :tREGEXP_END, "", :expr_end)
2236
+ :tREGEXP_BEG, "/", EXPR_BEG,
2237
+ :tSTRING_CONTENT, "regex\\C-x", EXPR_BEG,
2238
+ :tREGEXP_END, "", EXPR_LIT)
1983
2239
  end
1984
2240
 
1985
2241
  def test_yylex_regexp_escape_C_M
1986
2242
  assert_lex3("/regex\\C-\\M-x/",
1987
2243
  nil,
1988
- :tREGEXP_BEG, "/", :expr_beg,
1989
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1990
- :tREGEXP_END, "", :expr_end)
2244
+ :tREGEXP_BEG, "/", EXPR_BEG,
2245
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2246
+ :tREGEXP_END, "", EXPR_LIT)
1991
2247
  end
1992
2248
 
1993
2249
  def test_yylex_regexp_escape_C_M_craaaazy
1994
- assert_lex3("/regex\\C-\\\n\\M-x/",
2250
+ rb = "/regex\\C-\\\n\\M-x/"
2251
+ assert_lex3(rb,
1995
2252
  nil,
1996
- :tREGEXP_BEG, "/", :expr_beg,
1997
- :tSTRING_CONTENT, "regex\\C-\\M-x", :expr_beg,
1998
- :tREGEXP_END, "", :expr_end)
2253
+ :tREGEXP_BEG, "/", EXPR_BEG,
2254
+ :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
2255
+ :tREGEXP_END, "", EXPR_LIT)
1999
2256
  end
2000
2257
 
2001
2258
  def test_yylex_regexp_escape_C_bad_dash
@@ -2021,17 +2278,17 @@ class TestRubyLexer < Minitest::Test
2021
2278
  def test_yylex_regexp_escape_M
2022
2279
  assert_lex3("/regex\\M-x/",
2023
2280
  nil,
2024
- :tREGEXP_BEG, "/", :expr_beg,
2025
- :tSTRING_CONTENT, "regex\\M-x", :expr_beg,
2026
- :tREGEXP_END, "", :expr_end)
2281
+ :tREGEXP_BEG, "/", EXPR_BEG,
2282
+ :tSTRING_CONTENT, "regex\\M-x", EXPR_BEG,
2283
+ :tREGEXP_END, "", EXPR_LIT)
2027
2284
  end
2028
2285
 
2029
2286
  def test_yylex_regexp_escape_M_C
2030
2287
  assert_lex3("/regex\\M-\\C-x/",
2031
2288
  nil,
2032
- :tREGEXP_BEG, "/", :expr_beg,
2033
- :tSTRING_CONTENT, "regex\\M-\\C-x", :expr_beg,
2034
- :tREGEXP_END, "", :expr_end)
2289
+ :tREGEXP_BEG, "/", EXPR_BEG,
2290
+ :tSTRING_CONTENT, "regex\\M-\\C-x", EXPR_BEG,
2291
+ :tREGEXP_END, "", EXPR_LIT)
2035
2292
  end
2036
2293
 
2037
2294
  def test_yylex_regexp_escape_M_bad_dash
@@ -2053,49 +2310,45 @@ class TestRubyLexer < Minitest::Test
2053
2310
  def test_yylex_regexp_escape_backslash_slash
2054
2311
  assert_lex3("/\\//",
2055
2312
  nil,
2056
- :tREGEXP_BEG, "/", :expr_beg,
2057
- :tSTRING_CONTENT, "\\/", :expr_beg,
2058
- :tREGEXP_END, "", :expr_end)
2313
+ :tREGEXP_BEG, "/", EXPR_BEG,
2314
+ :tSTRING_CONTENT, "/", EXPR_BEG,
2315
+ :tREGEXP_END, "", EXPR_LIT)
2059
2316
  end
2060
2317
 
2061
2318
  def test_yylex_regexp_escape_backslash_terminator
2062
- assert_lex3("%r%blah\\%blah%",
2063
- nil,
2064
- :tREGEXP_BEG, "%r\000", :expr_beg,
2065
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2066
- :tREGEXP_END, "", :expr_end)
2067
- end
2068
-
2069
- def test_yylex_regexp_escaped_delim
2070
- assert_lex3("%r!blah(?\\!blah)!",
2071
- nil,
2072
- :tREGEXP_BEG, "%r\000", :expr_beg,
2073
- :tSTRING_CONTENT, "blah(?!blah)", :expr_beg,
2074
- :tREGEXP_END, "", :expr_end)
2319
+ rb = "%r%blah\\%blah%"
2320
+ assert_lex3(rb,
2321
+ s(:lit, /blah%blah/).line(1),
2322
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2323
+ :tSTRING_CONTENT, "blah%blah", EXPR_BEG,
2324
+ :tREGEXP_END, "", EXPR_LIT)
2075
2325
  end
2076
2326
 
2077
2327
  def test_yylex_regexp_escape_backslash_terminator_meta1
2078
2328
  assert_lex3("%r{blah\\}blah}",
2079
- nil,
2080
- :tREGEXP_BEG, "%r{", :expr_beg, # FIX ?!?
2081
- :tSTRING_CONTENT, "blah\\}blah", :expr_beg,
2082
- :tREGEXP_END, "", :expr_end)
2329
+ s(:lit, /blah\}blah/).line(1),
2330
+ :tREGEXP_BEG, "%r{", EXPR_BEG,
2331
+ :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2332
+ :tREGEXP_END, "", EXPR_LIT)
2083
2333
  end
2084
2334
 
2085
2335
  def test_yylex_regexp_escape_backslash_terminator_meta2
2086
- assert_lex3("%r/blah\\/blah/",
2087
- nil,
2088
- :tREGEXP_BEG, "%r\000", :expr_beg,
2089
- :tSTRING_CONTENT, "blah\\/blah", :expr_beg,
2090
- :tREGEXP_END, "", :expr_end)
2336
+ rb = "%r/blah\\/blah/"
2337
+ pt = s(:lit, /blah\/blah/).line 1
2338
+
2339
+ assert_lex3(rb,
2340
+ pt,
2341
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2342
+ :tSTRING_CONTENT, "blah/blah", EXPR_BEG,
2343
+ :tREGEXP_END, "", EXPR_LIT)
2091
2344
  end
2092
2345
 
2093
2346
  def test_yylex_regexp_escape_backslash_terminator_meta3
2094
2347
  assert_lex3("%r/blah\\%blah/",
2095
2348
  nil,
2096
- :tREGEXP_BEG, "%r\000", :expr_beg,
2097
- :tSTRING_CONTENT, "blah\\%blah", :expr_beg,
2098
- :tREGEXP_END, "", :expr_end)
2349
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2350
+ :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2351
+ :tREGEXP_END, "", EXPR_LIT)
2099
2352
  end
2100
2353
 
2101
2354
  def test_yylex_regexp_escape_bad_eos
@@ -2103,52 +2356,55 @@ class TestRubyLexer < Minitest::Test
2103
2356
  end
2104
2357
 
2105
2358
  def test_yylex_regexp_escape_bs
2106
- assert_lex3("/regex\\\\regex/",
2107
- nil,
2108
- :tREGEXP_BEG, "/", :expr_beg,
2109
- :tSTRING_CONTENT, "regex\\\\regex", :expr_beg,
2110
- :tREGEXP_END, "", :expr_end)
2359
+ rp = "/regex\\\\regex/"
2360
+ assert_lex3(rp,
2361
+ s(:lit, /regex\\regex/),
2362
+ :tREGEXP_BEG, "/", EXPR_BEG,
2363
+ :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2364
+ :tREGEXP_END, "", EXPR_LIT)
2111
2365
  end
2112
2366
 
2113
2367
  def test_yylex_regexp_escape_c
2114
2368
  assert_lex3("/regex\\cxxx/",
2115
2369
  nil,
2116
- :tREGEXP_BEG, "/", :expr_beg,
2117
- :tSTRING_CONTENT, "regex\\cxxx", :expr_beg,
2118
- :tREGEXP_END, "", :expr_end)
2370
+ :tREGEXP_BEG, "/", EXPR_BEG,
2371
+ :tSTRING_CONTENT, "regex\\cxxx", EXPR_BEG,
2372
+ :tREGEXP_END, "", EXPR_LIT)
2119
2373
  end
2120
2374
 
2121
2375
  def test_yylex_regexp_escape_c_backslash
2122
2376
  assert_lex3("/regex\\c\\n/",
2123
2377
  nil,
2124
- :tREGEXP_BEG, "/", :expr_beg,
2125
- :tSTRING_CONTENT, "regex\\c\\n", :expr_beg,
2126
- :tREGEXP_END, "", :expr_end)
2378
+ :tREGEXP_BEG, "/", EXPR_BEG,
2379
+ :tSTRING_CONTENT, "regex\\c\\n", EXPR_BEG,
2380
+ :tREGEXP_END, "", EXPR_LIT)
2127
2381
  end
2128
2382
 
2129
2383
  def test_yylex_regexp_escape_chars
2130
2384
  assert_lex3("/re\\tge\\nxp/",
2131
2385
  nil,
2132
- :tREGEXP_BEG, "/", :expr_beg,
2133
- :tSTRING_CONTENT, "re\\tge\\nxp", :expr_beg,
2134
- :tREGEXP_END, "", :expr_end)
2386
+ :tREGEXP_BEG, "/", EXPR_BEG,
2387
+ :tSTRING_CONTENT, "re\\tge\\nxp", EXPR_BEG,
2388
+ :tREGEXP_END, "", EXPR_LIT)
2135
2389
  end
2136
2390
 
2137
2391
  def test_yylex_regexp_escape_double_backslash
2138
- regexp = '/[\\/\\\\]$/'
2139
- assert_lex3(regexp.dup,
2140
- nil,
2141
- :tREGEXP_BEG, "/", :expr_beg,
2142
- :tSTRING_CONTENT, "[\\/\\\\]$", :expr_beg,
2143
- :tREGEXP_END, "", :expr_end)
2392
+ rb = '/[\\/\\\\]$/'
2393
+ pt = s(:lit, /[\/\\]$/)
2394
+
2395
+ assert_lex3(rb,
2396
+ pt,
2397
+ :tREGEXP_BEG, "/", EXPR_BEG,
2398
+ :tSTRING_CONTENT, "[/\\\\]$", EXPR_BEG,
2399
+ :tREGEXP_END, "", EXPR_LIT)
2144
2400
  end
2145
2401
 
2146
2402
  def test_yylex_regexp_escape_hex
2147
2403
  assert_lex3("/regex\\x61xp/",
2148
2404
  nil,
2149
- :tREGEXP_BEG, "/", :expr_beg,
2150
- :tSTRING_CONTENT, "regex\\x61xp", :expr_beg,
2151
- :tREGEXP_END, "", :expr_end)
2405
+ :tREGEXP_BEG, "/", EXPR_BEG,
2406
+ :tSTRING_CONTENT, "regex\\x61xp", EXPR_BEG,
2407
+ :tREGEXP_END, "", EXPR_LIT)
2152
2408
  end
2153
2409
 
2154
2410
  def test_yylex_regexp_escape_hex_bad
@@ -2158,558 +2414,666 @@ class TestRubyLexer < Minitest::Test
2158
2414
  def test_yylex_regexp_escape_hex_one
2159
2415
  assert_lex3("/^[\\xd\\xa]{2}/on",
2160
2416
  nil,
2161
- :tREGEXP_BEG, "/", :expr_beg,
2162
- :tSTRING_CONTENT, "^[\\xd\\xa]{2}", :expr_beg,
2163
- :tREGEXP_END, "on", :expr_end)
2417
+ :tREGEXP_BEG, "/", EXPR_BEG,
2418
+ :tSTRING_CONTENT, "^[\\xd\\xa]{2}", EXPR_BEG,
2419
+ :tREGEXP_END, "on", EXPR_LIT)
2164
2420
  end
2165
2421
 
2166
2422
  def test_yylex_regexp_escape_oct1
2167
2423
  assert_lex3("/regex\\0xp/",
2168
2424
  nil,
2169
- :tREGEXP_BEG, "/", :expr_beg,
2170
- :tSTRING_CONTENT, "regex\\0xp", :expr_beg,
2171
- :tREGEXP_END, "", :expr_end)
2425
+ :tREGEXP_BEG, "/", EXPR_BEG,
2426
+ :tSTRING_CONTENT, "regex\\0xp", EXPR_BEG,
2427
+ :tREGEXP_END, "", EXPR_LIT)
2172
2428
  end
2173
2429
 
2174
2430
  def test_yylex_regexp_escape_oct2
2175
2431
  assert_lex3("/regex\\07xp/",
2176
2432
  nil,
2177
- :tREGEXP_BEG, "/", :expr_beg,
2178
- :tSTRING_CONTENT, "regex\\07xp", :expr_beg,
2179
- :tREGEXP_END, "", :expr_end)
2433
+ :tREGEXP_BEG, "/", EXPR_BEG,
2434
+ :tSTRING_CONTENT, "regex\\07xp", EXPR_BEG,
2435
+ :tREGEXP_END, "", EXPR_LIT)
2180
2436
  end
2181
2437
 
2182
2438
  def test_yylex_regexp_escape_oct3
2183
2439
  assert_lex3("/regex\\10142/",
2184
2440
  nil,
2185
- :tREGEXP_BEG, "/", :expr_beg,
2186
- :tSTRING_CONTENT, "regex\\10142", :expr_beg,
2187
- :tREGEXP_END, "", :expr_end)
2441
+ :tREGEXP_BEG, "/", EXPR_BEG,
2442
+ :tSTRING_CONTENT, "regex\\10142", EXPR_BEG,
2443
+ :tREGEXP_END, "", EXPR_LIT)
2188
2444
  end
2189
2445
 
2190
2446
  def test_yylex_regexp_escape_return
2191
2447
  assert_lex3("/regex\\\nregex/",
2192
2448
  nil,
2193
- :tREGEXP_BEG, "/", :expr_beg,
2194
- :tSTRING_CONTENT, "regexregex", :expr_beg,
2195
- :tREGEXP_END, "", :expr_end)
2449
+ :tREGEXP_BEG, "/", EXPR_BEG,
2450
+ :tSTRING_CONTENT, "regexregex", EXPR_BEG,
2451
+ :tREGEXP_END, "", EXPR_LIT)
2452
+ end
2453
+
2454
+ def test_yylex_regexp_escaped_delim
2455
+ assert_lex3("%r!blah(?\\!blah)!",
2456
+ nil,
2457
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2458
+ :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2459
+ :tREGEXP_END, "", EXPR_LIT)
2196
2460
  end
2197
2461
 
2198
2462
  def test_yylex_regexp_nm
2199
2463
  assert_lex3("/.*/nm",
2200
2464
  nil,
2201
- :tREGEXP_BEG, "/", :expr_beg,
2202
- :tSTRING_CONTENT, ".*", :expr_beg,
2203
- :tREGEXP_END, "nm", :expr_end)
2465
+ :tREGEXP_BEG, "/", EXPR_BEG,
2466
+ :tSTRING_CONTENT, ".*", EXPR_BEG,
2467
+ :tREGEXP_END, "nm", EXPR_LIT)
2468
+ end
2469
+
2470
+ def test_yylex_required_kwarg_no_value_22
2471
+ setup_lexer_class RubyParser::V22
2472
+
2473
+ assert_lex3("def foo a:, b:\nend",
2474
+ nil,
2475
+ :kDEF, "def", EXPR_FNAME,
2476
+ :tIDENTIFIER, "foo", EXPR_ENDFN,
2477
+ :tLABEL, "a", EXPR_LAB,
2478
+ :tCOMMA, ",", EXPR_PAR,
2479
+ :tLABEL, "b", EXPR_LAB,
2480
+ :kEND, "end", EXPR_END)
2204
2481
  end
2205
2482
 
2206
2483
  def test_yylex_rparen
2207
- assert_lex3(")", nil, :tRPAREN, ")", :expr_endfn)
2484
+ assert_lex3(")", nil, :tRPAREN, ")", EXPR_ENDFN)
2208
2485
  end
2209
2486
 
2210
2487
  def test_yylex_rshft
2211
2488
  assert_lex3("a >> 2",
2212
2489
  nil,
2213
- :tIDENTIFIER, "a", :expr_cmdarg,
2214
- :tRSHFT, ">>", :expr_beg,
2215
- :tINTEGER, 2, :expr_end)
2490
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2491
+ :tRSHFT, ">>", EXPR_BEG,
2492
+ :tINTEGER, 2, EXPR_NUM)
2216
2493
  end
2217
2494
 
2218
2495
  def test_yylex_rshft_equals
2219
2496
  assert_lex3("a >>= 2",
2220
2497
  nil,
2221
- :tIDENTIFIER, "a", :expr_cmdarg,
2222
- :tOP_ASGN, ">>", :expr_beg,
2223
- :tINTEGER, 2, :expr_end)
2498
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2499
+ :tOP_ASGN, ">>", EXPR_BEG,
2500
+ :tINTEGER, 2, EXPR_NUM)
2224
2501
  end
2225
2502
 
2226
2503
  def test_yylex_star
2227
2504
  assert_lex3("a * ",
2228
2505
  nil,
2229
- :tIDENTIFIER, "a", :expr_cmdarg,
2230
- :tSTAR2, "*", :expr_beg)
2506
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2507
+ :tSTAR2, "*", EXPR_BEG)
2231
2508
  end
2232
2509
 
2233
2510
  def test_yylex_star2
2234
2511
  assert_lex3("a ** ",
2235
2512
  nil,
2236
- :tIDENTIFIER, "a", :expr_cmdarg,
2237
- :tPOW, "**", :expr_beg)
2513
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2514
+ :tPOW, "**", EXPR_BEG)
2238
2515
  end
2239
2516
 
2240
2517
  def test_yylex_star2_equals
2241
2518
  assert_lex3("a **= ",
2242
2519
  nil,
2243
- :tIDENTIFIER, "a", :expr_cmdarg,
2244
- :tOP_ASGN, "**", :expr_beg)
2520
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2521
+ :tOP_ASGN, "**", EXPR_BEG)
2245
2522
  end
2246
2523
 
2247
2524
  def test_yylex_star_arg
2248
- self.lex_state = :expr_arg
2525
+ self.lex_state = EXPR_ARG
2249
2526
 
2250
2527
  assert_lex3(" *a",
2251
2528
  nil,
2252
- :tSTAR, "*", :expr_beg,
2253
- :tIDENTIFIER, "a", :expr_arg)
2529
+ :tSTAR, "*", EXPR_BEG,
2530
+ :tIDENTIFIER, "a", EXPR_ARG)
2254
2531
  end
2255
2532
 
2256
2533
  def test_yylex_star_arg_beg
2257
- self.lex_state = :expr_beg
2534
+ self.lex_state = EXPR_BEG
2258
2535
 
2259
2536
  assert_lex3("*a",
2260
2537
  nil,
2261
- :tSTAR, "*", :expr_beg,
2262
- :tIDENTIFIER, "a", :expr_arg)
2538
+ :tSTAR, "*", EXPR_BEG,
2539
+ :tIDENTIFIER, "a", EXPR_ARG)
2263
2540
  end
2264
2541
 
2265
2542
  def test_yylex_star_arg_beg_fname
2266
- self.lex_state = :expr_fname
2543
+ self.lex_state = EXPR_FNAME
2267
2544
 
2268
2545
  assert_lex3("*a",
2269
2546
  nil,
2270
- :tSTAR2, "*", :expr_arg,
2271
- :tIDENTIFIER, "a", :expr_arg)
2547
+ :tSTAR2, "*", EXPR_ARG,
2548
+ :tIDENTIFIER, "a", EXPR_ARG)
2272
2549
  end
2273
2550
 
2274
2551
  def test_yylex_star_arg_beg_fname2
2275
- self.lex_state = :expr_fname
2552
+ self.lex_state = EXPR_FNAME
2276
2553
 
2277
2554
  assert_lex3("*a",
2278
2555
  nil,
2279
- :tSTAR2, "*", :expr_arg,
2280
- :tIDENTIFIER, "a", :expr_arg)
2556
+ :tSTAR2, "*", EXPR_ARG,
2557
+ :tIDENTIFIER, "a", EXPR_ARG)
2281
2558
  end
2282
2559
 
2283
2560
  def test_yylex_star_equals
2284
2561
  assert_lex3("a *= ",
2285
2562
  nil,
2286
- :tIDENTIFIER, "a", :expr_cmdarg,
2287
- :tOP_ASGN, "*", :expr_beg)
2563
+ :tIDENTIFIER, "a", EXPR_CMDARG,
2564
+ :tOP_ASGN, "*", EXPR_BEG)
2288
2565
  end
2289
2566
 
2290
2567
  def test_yylex_string_bad_eos
2291
- refute_lex('%', :tSTRING_BEG, '%')
2568
+ refute_lex("%", :tSTRING_BEG, "%")
2292
2569
  end
2293
2570
 
2294
2571
  def test_yylex_string_bad_eos_quote
2295
- refute_lex('%{nest', :tSTRING_BEG, '%}')
2572
+ refute_lex("%{nest",
2573
+ :tSTRING_BEG, "%}",
2574
+ :tSTRING_CONTENT, "nest")
2296
2575
  end
2297
2576
 
2298
2577
  def test_yylex_string_double
2299
- assert_lex3("\"string\"", nil, :tSTRING, "string", :expr_end)
2578
+ assert_lex3("\"string\"", nil, :tSTRING, "string", EXPR_END)
2300
2579
  end
2301
2580
 
2302
2581
  def test_yylex_string_double_escape_C
2303
- assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", :expr_end)
2582
+ assert_lex3("\"\\C-a\"", nil, :tSTRING, "\001", EXPR_END)
2304
2583
  end
2305
2584
 
2306
2585
  def test_yylex_string_double_escape_C_backslash
2307
- assert_lex3("\"\\C-\\\\\"",
2586
+ assert_lex3(%W[ " \\ C - \\ \\ " ].join, # I hate escaping \ in ' and "
2308
2587
  nil,
2309
- :tSTRING_BEG, "\"", :expr_beg,
2310
- :tSTRING_CONTENT, "\034", :expr_beg,
2311
- :tSTRING_END, "\"", :expr_end)
2588
+ :tSTRING_BEG, "\"", EXPR_BEG,
2589
+ :tSTRING_CONTENT, "\034", EXPR_BEG,
2590
+ :tSTRING_END, "\"", EXPR_LIT)
2312
2591
  end
2313
2592
 
2314
2593
  def test_yylex_string_double_escape_C_escape
2315
2594
  assert_lex3("\"\\C-\\M-a\"",
2316
2595
  nil,
2317
- :tSTRING_BEG, "\"", :expr_beg,
2318
- :tSTRING_CONTENT, "\201", :expr_beg,
2319
- :tSTRING_END, "\"", :expr_end)
2596
+ :tSTRING_BEG, "\"", EXPR_BEG,
2597
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2598
+ :tSTRING_END, "\"", EXPR_LIT)
2320
2599
  end
2321
2600
 
2322
2601
  def test_yylex_string_double_escape_C_question
2323
- assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", :expr_end)
2324
- end
2325
-
2326
- def test_yylex_string_utf8_simple
2327
- chr = [0x3024].pack("U")
2328
-
2329
- assert_lex3('"\u{3024}"',
2330
- s(:str, chr),
2331
- :tSTRING, chr, :expr_end)
2332
- end
2333
-
2334
- def test_yylex_string_utf8_complex
2335
- chr = [0x3024].pack("U")
2336
-
2337
- assert_lex3('"#@a\u{3024}"',
2338
- s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2339
- :tSTRING_BEG, '"', :expr_beg,
2340
- :tSTRING_DVAR, nil, :expr_beg,
2341
- :tSTRING_CONTENT, "@a"+chr, :expr_beg,
2342
- :tSTRING_END, '"', :expr_end)
2602
+ assert_lex3("\"\\C-?\"", nil, :tSTRING, "\177", EXPR_END)
2343
2603
  end
2344
2604
 
2345
2605
  def test_yylex_string_double_escape_M
2346
2606
  chr = "\341"
2347
- chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
2348
-
2349
- assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
2350
- end
2351
2607
 
2352
- def test_why_does_ruby_hate_me?
2353
- assert_lex3("\"Nl%\\000\\000A\\000\\999\"", # you should be ashamed
2354
- nil,
2355
- :tSTRING, ["Nl%","\x00","\x00","A","\x00","999"].join, :expr_end)
2608
+ assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, EXPR_END)
2356
2609
  end
2357
2610
 
2358
2611
  def test_yylex_string_double_escape_M_backslash
2359
2612
  assert_lex3("\"\\M-\\\\\"",
2360
2613
  nil,
2361
- :tSTRING_BEG, "\"", :expr_beg,
2362
- :tSTRING_CONTENT, "\334", :expr_beg,
2363
- :tSTRING_END, "\"", :expr_end)
2614
+ :tSTRING_BEG, "\"", EXPR_BEG,
2615
+ :tSTRING_CONTENT, "\334", EXPR_BEG,
2616
+ :tSTRING_END, "\"", EXPR_LIT)
2364
2617
  end
2365
2618
 
2366
2619
  def test_yylex_string_double_escape_M_escape
2367
2620
  assert_lex3("\"\\M-\\C-a\"",
2368
2621
  nil,
2369
- :tSTRING_BEG, "\"", :expr_beg,
2370
- :tSTRING_CONTENT, "\201", :expr_beg,
2371
- :tSTRING_END, "\"", :expr_end)
2622
+ :tSTRING_BEG, "\"", EXPR_BEG,
2623
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2624
+ :tSTRING_END, "\"", EXPR_LIT)
2372
2625
  end
2373
2626
 
2374
2627
  def test_yylex_string_double_escape_bs1
2375
- assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", :expr_end)
2628
+ assert_lex3("\"a\\a\\a\"", nil, :tSTRING, "a\a\a", EXPR_END)
2376
2629
  end
2377
2630
 
2378
2631
  def test_yylex_string_double_escape_bs2
2379
- assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", :expr_end)
2632
+ assert_lex3("\"a\\\\a\"", nil, :tSTRING, "a\\a", EXPR_END)
2380
2633
  end
2381
2634
 
2382
2635
  def test_yylex_string_double_escape_c
2383
- assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", :expr_end)
2636
+ assert_lex3("\"\\ca\"", nil, :tSTRING, "\001", EXPR_END)
2384
2637
  end
2385
2638
 
2386
2639
  def test_yylex_string_double_escape_c_backslash
2387
- assert_lex3("\"\\c\\\"",
2388
- nil,
2389
- :tSTRING_BEG, "\"", :expr_beg,
2390
- :tSTRING_CONTENT, "\034", :expr_beg,
2391
- :tSTRING_END, "\"", :expr_end)
2640
+ refute_lex('"\\c\\"',
2641
+ :tSTRING_BEG, '"',
2642
+ :tSTRING_CONTENT, "\002")
2392
2643
  end
2393
2644
 
2394
2645
  def test_yylex_string_double_escape_c_escape
2395
2646
  assert_lex3("\"\\c\\M-a\"",
2396
2647
  nil,
2397
- :tSTRING_BEG, "\"", :expr_beg,
2398
- :tSTRING_CONTENT, "\201", :expr_beg,
2399
- :tSTRING_END, "\"", :expr_end)
2648
+ :tSTRING_BEG, "\"", EXPR_BEG,
2649
+ :tSTRING_CONTENT, "\201", EXPR_BEG,
2650
+ :tSTRING_END, "\"", EXPR_LIT)
2400
2651
  end
2401
2652
 
2402
2653
  def test_yylex_string_double_escape_c_question
2403
- assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", :expr_end)
2654
+ assert_lex3("\"\\c?\"", nil, :tSTRING, "\177", EXPR_END)
2404
2655
  end
2405
2656
 
2406
2657
  def test_yylex_string_double_escape_chars
2407
- assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", :expr_end)
2658
+ assert_lex3("\"s\\tri\\ng\"", nil, :tSTRING, "s\tri\ng", EXPR_END)
2408
2659
  end
2409
2660
 
2410
2661
  def test_yylex_string_double_escape_hex
2411
- assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", :expr_end)
2662
+ assert_lex3("\"n = \\x61\\x62\\x63\"", nil, :tSTRING, "n = abc", EXPR_END)
2412
2663
  end
2413
2664
 
2414
2665
  def test_yylex_string_double_escape_octal
2415
- assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", :expr_end)
2666
+ assert_lex3("\"n = \\101\\102\\103\"", nil, :tSTRING, "n = ABC", EXPR_END)
2416
2667
  end
2417
2668
 
2418
2669
  def test_yylex_string_double_escape_octal_fucked
2419
- assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", :expr_end)
2670
+ assert_lex3("\"n = \\444\"", nil, :tSTRING, "n = $", EXPR_END)
2420
2671
  end
2421
2672
 
2422
2673
  def test_yylex_string_double_interp
2423
2674
  assert_lex3("\"blah #x a \#@a b \#$b c \#{3} # \"",
2424
2675
  nil,
2425
- :tSTRING_BEG, "\"", :expr_beg,
2426
- :tSTRING_CONTENT, "blah #x a ", :expr_beg,
2427
- :tSTRING_DVAR, nil, :expr_beg,
2428
- :tSTRING_CONTENT, "@a b ", :expr_beg,
2429
- :tSTRING_DVAR, nil, :expr_beg,
2430
- :tSTRING_CONTENT, "$b c ", :expr_beg,
2431
- :tSTRING_DBEG, nil, :expr_beg,
2432
- :tSTRING_CONTENT, "3} # ", :expr_beg,
2433
- :tSTRING_END, "\"", :expr_end)
2434
- end
2435
-
2436
- def test_yylex_string_double_pound_dollar_bad
2437
- skip if ruby18
2438
-
2439
- assert_lex3('"#$%"', nil,
2440
-
2441
- :tSTRING_BEG, "\"", :expr_beg,
2442
- :tSTRING_CONTENT, '#$%', :expr_beg,
2443
- :tSTRING_END, "\"", :expr_end)
2676
+ :tSTRING_BEG, "\"", EXPR_BEG,
2677
+ :tSTRING_CONTENT, "blah #x a ", EXPR_BEG,
2678
+ :tSTRING_DVAR, "#", EXPR_BEG,
2679
+ :tSTRING_CONTENT, "@a b ", EXPR_BEG,
2680
+ :tSTRING_DVAR, "#", EXPR_BEG,
2681
+ :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2682
+ :tSTRING_DBEG, "#\{", EXPR_BEG,
2683
+ :tSTRING_CONTENT, "3} # ", EXPR_BEG, # FIX: wrong!?!?
2684
+ :tSTRING_END, "\"", EXPR_LIT)
2444
2685
  end
2445
2686
 
2446
2687
  def test_yylex_string_double_nested_curlies
2447
2688
  assert_lex3("%{nest{one{two}one}nest}",
2448
2689
  nil,
2449
- :tSTRING_BEG, "%}", :expr_beg,
2450
- :tSTRING_CONTENT, "nest{one{two}one}nest", :expr_beg,
2451
- :tSTRING_END, "}", :expr_end)
2690
+ :tSTRING_BEG, "%}", EXPR_BEG,
2691
+ :tSTRING_CONTENT, "nest{one{two}one}nest", EXPR_BEG,
2692
+ :tSTRING_END, "}", EXPR_LIT)
2452
2693
  end
2453
2694
 
2454
2695
  def test_yylex_string_double_no_interp
2455
- assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", :expr_end)
2456
- assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", :expr_end)
2696
+ assert_lex3("\"# blah\"", nil, :tSTRING, "# blah", EXPR_END)
2697
+ assert_lex3("\"blah # blah\"", nil, :tSTRING, "blah # blah", EXPR_END)
2457
2698
  end
2458
2699
 
2459
- def test_yylex_string_escape_x_single
2460
- assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", :expr_end)
2700
+ def test_yylex_string_double_pound_dollar_bad
2701
+ assert_lex3('"#$%"', nil,
2702
+
2703
+ :tSTRING_BEG, "\"", EXPR_BEG,
2704
+ :tSTRING_CONTENT, "#\$%", EXPR_BEG,
2705
+ :tSTRING_END, "\"", EXPR_LIT)
2461
2706
  end
2462
2707
 
2463
- def test_yylex_string_pct_i
2464
- assert_lex3("%i[s1 s2\ns3]",
2465
- nil,
2466
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2467
- :tSTRING_CONTENT, "s1", :expr_beg,
2468
- :tSPACE, nil, :expr_beg,
2469
- :tSTRING_CONTENT, "s2", :expr_beg,
2470
- :tSPACE, nil, :expr_beg,
2471
- :tSTRING_CONTENT, "s3", :expr_beg,
2472
- :tSPACE, nil, :expr_beg,
2473
- :tSTRING_END, nil, :expr_end)
2708
+ def test_yylex_string_escape_x_single
2709
+ assert_lex3("\"\\x0\"", nil, :tSTRING, "\000", EXPR_END)
2474
2710
  end
2475
2711
 
2476
2712
  def test_yylex_string_pct_I
2477
2713
  assert_lex3("%I[s1 s2\ns3]",
2478
2714
  nil,
2479
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2480
- :tSTRING_CONTENT, "s1", :expr_beg,
2481
- :tSPACE, nil, :expr_beg,
2482
- :tSTRING_CONTENT, "s2", :expr_beg,
2483
- :tSPACE, nil, :expr_beg,
2484
- :tSTRING_CONTENT, "s3", :expr_beg,
2485
- :tSPACE, nil, :expr_beg,
2486
- :tSTRING_END, nil, :expr_end)
2715
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2716
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2717
+ :tSPACE, " ", EXPR_BEG,
2718
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2719
+ :tSPACE, " ", EXPR_BEG,
2720
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2721
+ :tSPACE, "]", EXPR_BEG,
2722
+ :tSTRING_END, "]", EXPR_LIT)
2487
2723
  end
2488
2724
 
2489
- def test_yylex_string_pct_i_extra_space
2490
- assert_lex3("%i[ s1 s2\ns3 ]",
2725
+ def test_yylex_string_pct_I_extra_space
2726
+ assert_lex3("%I[ s1 s2\ns3 ]",
2491
2727
  nil,
2492
- :tQSYMBOLS_BEG, "%i[", :expr_beg,
2493
- :tSTRING_CONTENT, "s1", :expr_beg,
2494
- :tSPACE, nil, :expr_beg,
2495
- :tSTRING_CONTENT, "s2", :expr_beg,
2496
- :tSPACE, nil, :expr_beg,
2497
- :tSTRING_CONTENT, "s3", :expr_beg,
2498
- :tSPACE, nil, :expr_beg,
2499
- :tSTRING_END, nil, :expr_end)
2728
+ :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2729
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2730
+ :tSPACE, " ", EXPR_BEG,
2731
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2732
+ :tSPACE, " ", EXPR_BEG,
2733
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2734
+ :tSPACE, "]", EXPR_BEG,
2735
+ :tSTRING_END, "]", EXPR_LIT)
2500
2736
  end
2501
2737
 
2502
- def test_yylex_string_pct_I_extra_space
2503
- assert_lex3("%I[ s1 s2\ns3 ]",
2738
+ def test_yylex_string_pct_Q
2739
+ assert_lex3("%Q[s1 s2]",
2504
2740
  nil,
2505
- :tSYMBOLS_BEG, "%I[", :expr_beg,
2506
- :tSTRING_CONTENT, "s1", :expr_beg,
2507
- :tSPACE, nil, :expr_beg,
2508
- :tSTRING_CONTENT, "s2", :expr_beg,
2509
- :tSPACE, nil, :expr_beg,
2510
- :tSTRING_CONTENT, "s3", :expr_beg,
2511
- :tSPACE, nil, :expr_beg,
2512
- :tSTRING_END, nil, :expr_end)
2741
+ :tSTRING_BEG, "%Q[", EXPR_BEG,
2742
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2743
+ :tSTRING_END, "]", EXPR_LIT)
2513
2744
  end
2514
2745
 
2515
- def test_yylex_string_pct_q
2516
- assert_lex3("%q[s1 s2]",
2746
+ def test_yylex_string_pct_Q_null_wtf?
2747
+ assert_lex3("%Q\0s1 s2\0",
2517
2748
  nil,
2518
- :tSTRING_BEG, "%q[", :expr_beg,
2519
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2520
- :tSTRING_END, "]", :expr_end)
2749
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2750
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2751
+ :tSTRING_END, "\0", EXPR_LIT)
2521
2752
  end
2522
2753
 
2523
- def test_yylex_string_pct_Q
2524
- assert_lex3("%Q[s1 s2]",
2754
+ def test_yylex_string_pct_Q_bang
2755
+ assert_lex3("%Q!s1 s2!",
2525
2756
  nil,
2526
- :tSTRING_BEG, "%Q[", :expr_beg,
2527
- :tSTRING_CONTENT, "s1 s2", :expr_beg,
2528
- :tSTRING_END, "]", :expr_end)
2757
+ :tSTRING_BEG, "%Q\0", EXPR_BEG,
2758
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2759
+ :tSTRING_END, "!", EXPR_LIT)
2529
2760
  end
2530
2761
 
2531
2762
  def test_yylex_string_pct_W
2532
2763
  assert_lex3("%W[s1 s2\ns3]", # TODO: add interpolation to these
2533
2764
  nil,
2534
- :tWORDS_BEG, "%W[", :expr_beg,
2535
- :tSTRING_CONTENT, "s1", :expr_beg,
2536
- :tSPACE, nil, :expr_beg,
2537
- :tSTRING_CONTENT, "s2", :expr_beg,
2538
- :tSPACE, nil, :expr_beg,
2539
- :tSTRING_CONTENT, "s3", :expr_beg,
2540
- :tSPACE, nil, :expr_beg,
2541
- :tSTRING_END, nil, :expr_end)
2765
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2766
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2767
+ :tSPACE, " ", EXPR_BEG,
2768
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2769
+ :tSPACE, " ", EXPR_BEG,
2770
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2771
+ :tSPACE, "]", EXPR_BEG,
2772
+ :tSTRING_END, "]", EXPR_LIT)
2542
2773
  end
2543
2774
 
2544
2775
  def test_yylex_string_pct_W_bs_nl
2545
- assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2546
- nil,
2547
- :tWORDS_BEG, "%W[", :expr_beg,
2548
- :tSTRING_CONTENT, "s1", :expr_beg,
2549
- :tSPACE, nil, :expr_beg,
2550
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2551
- :tSPACE, nil, :expr_beg,
2552
- :tSTRING_END, nil, :expr_end)
2776
+ rb = "%W[s1 \\\ns2]" # TODO: add interpolation to these
2777
+ pt = s(:array,
2778
+ s(:str, "s1").line(1),
2779
+ s(:str, "\ns2").line(1)).line(1)
2780
+
2781
+ assert_lex3(rb,
2782
+ pt,
2783
+ :tWORDS_BEG, "%W[", EXPR_BEG,
2784
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2785
+ :tSPACE, " ", EXPR_BEG,
2786
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2787
+ :tSPACE, "]", EXPR_BEG,
2788
+ :tSTRING_END, "]", EXPR_LIT)
2553
2789
  end
2554
2790
 
2555
2791
  def test_yylex_string_pct_angle
2556
2792
  assert_lex3("%<blah>",
2557
2793
  nil,
2558
- :tSTRING_BEG, "%>", :expr_beg,
2559
- :tSTRING_CONTENT, "blah", :expr_beg,
2560
- :tSTRING_END, ">", :expr_end)
2794
+ :tSTRING_BEG, "%>", EXPR_BEG,
2795
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2796
+ :tSTRING_END, ">", EXPR_LIT)
2797
+ end
2798
+
2799
+ def test_yylex_string_pct_i
2800
+ assert_lex3("%i[s1 s2\ns3]",
2801
+ nil,
2802
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2803
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2804
+ :tSPACE, " ", EXPR_BEG,
2805
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2806
+ :tSPACE, " ", EXPR_BEG,
2807
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2808
+ :tSPACE, "]", EXPR_BEG,
2809
+ :tSTRING_END, "]", EXPR_LIT)
2810
+ end
2811
+
2812
+ def test_yylex_string_pct_i_extra_space
2813
+ assert_lex3("%i[ s1 s2\ns3 ]",
2814
+ nil,
2815
+ :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2816
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2817
+ :tSPACE, " ", EXPR_BEG,
2818
+ :tSTRING_CONTENT, "s2", EXPR_BEG,
2819
+ :tSPACE, " ", EXPR_BEG,
2820
+ :tSTRING_CONTENT, "s3", EXPR_BEG,
2821
+ :tSPACE, "]", EXPR_BEG,
2822
+ :tSTRING_END, "]", EXPR_LIT)
2561
2823
  end
2562
2824
 
2563
2825
  def test_yylex_string_pct_other
2564
2826
  assert_lex3("%%blah%",
2565
2827
  nil,
2566
- :tSTRING_BEG, "%%", :expr_beg,
2567
- :tSTRING_CONTENT, "blah", :expr_beg,
2568
- :tSTRING_END, "%", :expr_end)
2828
+ :tSTRING_BEG, "%%", EXPR_BEG,
2829
+ :tSTRING_CONTENT, "blah", EXPR_BEG,
2830
+ :tSTRING_END, "%", EXPR_LIT)
2831
+ end
2832
+
2833
+ def test_yylex_string_pct_q
2834
+ assert_lex3("%q[s1 s2]",
2835
+ nil,
2836
+ :tSTRING_BEG, "%q[", EXPR_BEG,
2837
+ :tSTRING_CONTENT, "s1 s2", EXPR_BEG,
2838
+ :tSTRING_END, "]", EXPR_LIT)
2839
+ end
2840
+
2841
+ def test_yylex_string_pct_s
2842
+ assert_lex3("%s[s1 s2]",
2843
+ nil,
2844
+ :tSYMBEG, "%s[", EXPR_FNAME, # TODO: :tSYM_BEG ?
2845
+ :tSTRING_CONTENT, "s1 s2", EXPR_FNAME, # man... I don't like this
2846
+ :tSTRING_END, "]", EXPR_LIT)
2569
2847
  end
2570
2848
 
2571
2849
  def test_yylex_string_pct_w
2572
2850
  refute_lex("%w[s1 s2 ",
2573
2851
  :tQWORDS_BEG, "%w[",
2574
2852
  :tSTRING_CONTENT, "s1",
2575
- :tSPACE, nil,
2853
+ :tSPACE, " ",
2576
2854
  :tSTRING_CONTENT, "s2",
2577
- :tSPACE, nil)
2855
+ :tSPACE, " ")
2578
2856
  end
2579
2857
 
2580
2858
  def test_yylex_string_pct_w_bs_nl
2581
2859
  assert_lex3("%w[s1 \\\ns2]",
2582
2860
  nil,
2583
- :tQWORDS_BEG, "%w[", :expr_beg,
2584
- :tSTRING_CONTENT, "s1", :expr_beg,
2585
- :tSPACE, nil, :expr_beg,
2586
- :tSTRING_CONTENT, "\ns2", :expr_beg,
2587
- :tSPACE, nil, :expr_beg,
2588
- :tSTRING_END, nil, :expr_end)
2861
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2862
+ :tSTRING_CONTENT, "s1", EXPR_BEG,
2863
+ :tSPACE, " ", EXPR_BEG,
2864
+ :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2865
+ :tSPACE, "]", EXPR_BEG,
2866
+ :tSTRING_END, "]", EXPR_LIT)
2589
2867
  end
2590
2868
 
2591
2869
  def test_yylex_string_pct_w_bs_sp
2592
2870
  assert_lex3("%w[s\\ 1 s\\ 2]",
2593
- nil,
2594
- :tQWORDS_BEG, "%w[", :expr_beg,
2595
- :tSTRING_CONTENT, "s 1", :expr_beg,
2596
- :tSPACE, nil, :expr_beg,
2597
- :tSTRING_CONTENT, "s 2", :expr_beg,
2598
- :tSPACE, nil, :expr_beg,
2599
- :tSTRING_END, nil, :expr_end)
2871
+ s(:array, s(:str, "s 1"), s(:str, "s 2")),
2872
+ :tQWORDS_BEG, "%w[", EXPR_BEG,
2873
+ :tSTRING_CONTENT, "s 1", EXPR_BEG,
2874
+ :tSPACE, " ", EXPR_BEG,
2875
+ :tSTRING_CONTENT, "s 2", EXPR_BEG,
2876
+ :tSPACE, "]", EXPR_BEG,
2877
+ :tSTRING_END, "]", EXPR_LIT)
2600
2878
  end
2601
2879
 
2602
2880
  def test_yylex_string_single
2603
- assert_lex3("'string'", nil, :tSTRING, "string", :expr_end)
2881
+ assert_lex3("'string'", nil, :tSTRING, "string", EXPR_END)
2604
2882
  end
2605
2883
 
2606
2884
  def test_yylex_string_single_escape_chars
2607
- assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", :expr_end)
2885
+ assert_lex3("'s\\tri\\ng'", nil, :tSTRING, "s\\tri\\ng", EXPR_END)
2608
2886
  end
2609
2887
 
2610
- def test_yylex_string_single_nl
2611
- assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", :expr_end)
2888
+ def test_yylex_string_single_escape_quote_and_backslash
2889
+ assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz",
2890
+ EXPR_LIT)
2612
2891
  end
2613
2892
 
2614
2893
  def test_yylex_string_single_escaped_quote
2615
- assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", :expr_end)
2894
+ assert_lex3("'foo\\'bar'", nil, :tSTRING, "foo'bar", EXPR_END)
2616
2895
  end
2617
2896
 
2618
- def test_yylex_symbol
2619
- assert_lex3(":symbol", nil, :tSYMBOL, "symbol", :expr_end)
2897
+ def test_yylex_string_single_nl
2898
+ assert_lex3("'blah\\\nblah'", nil, :tSTRING, "blah\\\nblah", EXPR_END)
2620
2899
  end
2621
2900
 
2622
- def test_yylex_symbol_zero_byte__18
2623
- setup_lexer_class RubyParser::V18
2901
+ def test_yylex_string_utf8_complex
2902
+ chr = [0x3024].pack("U")
2624
2903
 
2625
- refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
2904
+ assert_lex3('"#@a\u{3024}"',
2905
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2906
+ :tSTRING_BEG, '"', EXPR_BEG,
2907
+ :tSTRING_DVAR, "#", EXPR_BEG,
2908
+ :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2909
+ :tSTRING_END, '"', EXPR_LIT)
2626
2910
  end
2627
2911
 
2628
- def test_yylex_symbol_zero_byte
2629
- assert_lex(":\"symbol\0\"", nil,
2630
- :tSYMBOL, "symbol\0", :expr_end)
2912
+ def test_yylex_string_utf8_complex_missing_hex
2913
+ chr = [0x302].pack("U")
2914
+ str = "#{chr}zzz"
2915
+
2916
+ refute_lex('"#@a\u302zzz"',
2917
+ :tSTRING_BEG, '"',
2918
+ :tSTRING_DVAR, "#",
2919
+ :tSTRING_CONTENT, "@a"+str,
2920
+ :tSTRING_END, '"')
2921
+
2922
+ chr = [0x30].pack("U")
2923
+ str = "#{chr}zzz"
2924
+
2925
+ refute_lex('"#@a\u30zzz"',
2926
+ :tSTRING_BEG, '"',
2927
+ :tSTRING_DVAR, "#",
2928
+ :tSTRING_CONTENT, "@a"+str,
2929
+ :tSTRING_END, '"')
2930
+
2931
+ chr = [0x3].pack("U")
2932
+ str = "#{chr}zzz"
2933
+
2934
+ refute_lex('"#@a\u3zzz"',
2935
+ :tSTRING_BEG, '"',
2936
+ :tSTRING_DVAR, "#",
2937
+ :tSTRING_CONTENT, "@a"+str,
2938
+ :tSTRING_END, '"')
2939
+ end
2940
+
2941
+ def test_yylex_string_utf8_bad_encoding_with_escapes
2942
+ str = "\"\\xBADπ\""
2943
+ exp = "\xBADπ".b
2944
+
2945
+ assert_lex(str,
2946
+ s(:str, exp),
2947
+ :tSTRING, exp, EXPR_END)
2948
+ end
2949
+
2950
+ def test_yylex_string_utf8_complex_trailing_hex
2951
+ chr = [0x3024].pack("U")
2952
+ str = "#{chr}abz"
2953
+
2954
+ assert_lex3('"#@a\u3024abz"',
2955
+ s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2956
+ :tSTRING_BEG, '"', EXPR_BEG,
2957
+ :tSTRING_DVAR, "#", EXPR_BEG,
2958
+ :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2959
+ :tSTRING_END, '"', EXPR_LIT)
2960
+ end
2961
+
2962
+ def test_yylex_string_utf8_missing_hex
2963
+ refute_lex('"\u3zzz"')
2964
+ refute_lex('"\u30zzz"')
2965
+ refute_lex('"\u302zzz"')
2966
+ end
2967
+
2968
+ def test_yylex_string_utf8_simple
2969
+ chr = [0x3024].pack("U")
2970
+
2971
+ assert_lex3('"\u{3024}"',
2972
+ s(:str, chr),
2973
+ :tSTRING, chr, EXPR_END)
2974
+ end
2975
+
2976
+ def test_yylex_string_utf8_trailing_hex
2977
+ chr = [0x3024].pack("U")
2978
+ str = "#{chr}abz"
2979
+
2980
+ assert_lex3('"\u3024abz"',
2981
+ s(:str, str),
2982
+ :tSTRING, str, EXPR_END)
2983
+ end
2984
+
2985
+ def test_yylex_sym_quoted
2986
+ assert_lex(":'a'",
2987
+ s(:lit, :a),
2988
+
2989
+ :tSYMBOL, "a", EXPR_LIT, 0, 0)
2990
+ end
2991
+
2992
+ def test_yylex_symbol
2993
+ assert_lex3(":symbol", nil, :tSYMBOL, "symbol", EXPR_LIT)
2631
2994
  end
2632
2995
 
2633
2996
  def test_yylex_symbol_double
2634
2997
  assert_lex3(":\"symbol\"",
2635
2998
  nil,
2636
- :tSYMBOL, "symbol", :expr_end)
2999
+ :tSYMBOL, "symbol", EXPR_LIT)
2637
3000
  end
2638
3001
 
2639
3002
  def test_yylex_symbol_double_interp
2640
3003
  assert_lex3(':"symbol#{1+1}"',
2641
3004
  nil,
2642
- :tSYMBEG, ":", :expr_fname,
2643
- :tSTRING_CONTENT, "symbol", :expr_fname,
2644
- :tSTRING_DBEG, nil, :expr_fname,
2645
- :tSTRING_CONTENT, "1+1}", :expr_fname, # HUH? this is BS
2646
- :tSTRING_END, "\"", :expr_end)
3005
+ :tSYMBEG, ":", EXPR_FNAME,
3006
+ :tSTRING_CONTENT, "symbol", EXPR_FNAME,
3007
+ :tSTRING_DBEG, '#{', EXPR_FNAME,
3008
+ :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
3009
+ :tSTRING_END, "\"", EXPR_LIT)
2647
3010
  end
2648
3011
 
2649
3012
  def test_yylex_symbol_single
2650
3013
  assert_lex3(":'symbol'",
2651
3014
  nil,
2652
- :tSYMBOL, "symbol", :expr_end)
3015
+ :tSYMBOL, "symbol", EXPR_LIT)
2653
3016
  end
2654
3017
 
2655
- def test_yylex_symbol_single_noninterp
2656
- assert_lex3(':\'symbol#{1+1}\'',
3018
+ def test_yylex_symbol_single_escape_chars
3019
+ assert_lex3(":'s\\tri\\ng'",
2657
3020
  nil,
2658
- :tSYMBOL, 'symbol#{1+1}', :expr_end)
3021
+ :tSYMBOL, "s\\tri\\ng", EXPR_LIT)
2659
3022
  end
2660
3023
 
2661
- def test_yylex_symbol_single_escape_chars
2662
- assert_lex3(":'s\\tri\\ng'",
3024
+ def test_yylex_symbol_single_noninterp
3025
+ assert_lex3(':\'symbol#{1+1}\'',
2663
3026
  nil,
2664
- :tSYMBOL, "s\\tri\\ng", :expr_end)
3027
+ :tSYMBOL, 'symbol#{1+1}', EXPR_LIT)
2665
3028
  end
2666
3029
 
2667
- def test_yylex_string_single_escape_quote_and_backslash
2668
- assert_lex3(":'foo\\'bar\\\\baz'", nil, :tSYMBOL, "foo'bar\\baz", :expr_end)
3030
+ def test_yylex_symbol_zero_byte
3031
+ assert_lex(":\"symbol\0\"", nil,
3032
+ :tSYMBOL, "symbol\0", EXPR_LIT)
2669
3033
  end
2670
3034
 
2671
3035
  def test_yylex_ternary1
2672
3036
  assert_lex3("a ? b : c",
2673
3037
  nil,
2674
- :tIDENTIFIER, "a", :expr_cmdarg,
2675
- :tEH, "?", :expr_value,
2676
- :tIDENTIFIER, "b", :expr_arg,
2677
- :tCOLON, ":", :expr_beg,
2678
- :tIDENTIFIER, "c", :expr_arg)
3038
+ :tIDENTIFIER, "a", EXPR_CMDARG,
3039
+ :tEH, "?", EXPR_BEG,
3040
+ :tIDENTIFIER, "b", EXPR_ARG,
3041
+ :tCOLON, ":", EXPR_BEG,
3042
+ :tIDENTIFIER, "c", EXPR_ARG)
2679
3043
 
2680
3044
  assert_lex3("a ?bb : c", # GAH! MATZ!!!
2681
3045
  nil,
2682
- :tIDENTIFIER, "a", :expr_cmdarg,
2683
- :tEH, "?", :expr_beg,
2684
- :tIDENTIFIER, "bb", :expr_arg,
2685
- :tCOLON, ":", :expr_beg,
2686
- :tIDENTIFIER, "c", :expr_arg)
3046
+ :tIDENTIFIER, "a", EXPR_CMDARG,
3047
+ :tEH, "?", EXPR_BEG,
3048
+ :tIDENTIFIER, "bb", EXPR_ARG,
3049
+ :tCOLON, ":", EXPR_BEG,
3050
+ :tIDENTIFIER, "c", EXPR_ARG)
2687
3051
 
2688
3052
  assert_lex3("42 ?",
2689
3053
  nil,
2690
- :tINTEGER, 42, :expr_end,
2691
- :tEH, "?", :expr_value)
3054
+ :tINTEGER, 42, EXPR_NUM,
3055
+ :tEH, "?", EXPR_BEG)
2692
3056
  end
2693
3057
 
2694
3058
  def test_yylex_tilde
2695
- assert_lex3("~", nil, :tTILDE, "~", :expr_beg)
3059
+ assert_lex3("~", nil, :tTILDE, "~", EXPR_BEG)
2696
3060
  end
2697
3061
 
2698
3062
  def test_yylex_tilde_unary
2699
- self.lex_state = :expr_fname
3063
+ self.lex_state = EXPR_FNAME
2700
3064
 
2701
- assert_lex3("~@", nil, :tTILDE, "~", :expr_arg)
3065
+ assert_lex3("~@", nil, :tTILDE, "~", EXPR_ARG)
2702
3066
  end
2703
3067
 
2704
3068
  def test_yylex_uminus
2705
3069
  assert_lex3("-blah",
2706
3070
  nil,
2707
- :tUMINUS, "-", :expr_beg,
2708
- :tIDENTIFIER, "blah", :expr_arg)
3071
+ :tUMINUS, "-", EXPR_BEG,
3072
+ :tIDENTIFIER, "blah", EXPR_ARG)
2709
3073
  end
2710
3074
 
2711
3075
  def test_yylex_underscore
2712
- assert_lex3("_var", nil, :tIDENTIFIER, "_var", :expr_cmdarg)
3076
+ assert_lex3("_var", nil, :tIDENTIFIER, "_var", EXPR_CMDARG)
2713
3077
  end
2714
3078
 
2715
3079
  def test_yylex_underscore_end
@@ -2721,230 +3085,45 @@ class TestRubyLexer < Minitest::Test
2721
3085
  def test_yylex_uplus
2722
3086
  assert_lex3("+blah",
2723
3087
  nil,
2724
- :tUPLUS, "+", :expr_beg,
2725
- :tIDENTIFIER, "blah", :expr_arg)
3088
+ :tUPLUS, "+", EXPR_BEG,
3089
+ :tIDENTIFIER, "blah", EXPR_ARG)
2726
3090
  end
2727
3091
 
2728
3092
  def test_zbug_float_in_decl
2729
3093
  assert_lex3("def initialize(u = 0.0, s = 0.0",
2730
3094
  nil,
2731
- :kDEF, "def", :expr_fname,
2732
- :tIDENTIFIER, "initialize", :expr_endfn,
2733
- :tLPAREN2, "(", :expr_beg,
2734
- :tIDENTIFIER, "u", :expr_arg,
2735
- :tEQL, "=", :expr_beg,
2736
- :tFLOAT, 0.0, :expr_end,
2737
- :tCOMMA, ",", :expr_beg,
2738
- :tIDENTIFIER, "s", :expr_arg,
2739
- :tEQL, "=", :expr_beg,
2740
- :tFLOAT, 0.0, :expr_end)
3095
+ :kDEF, "def", EXPR_FNAME,
3096
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3097
+ :tLPAREN2, "(", EXPR_PAR,
3098
+ :tIDENTIFIER, "u", EXPR_ARG,
3099
+ :tEQL, "=", EXPR_BEG,
3100
+ :tFLOAT, 0.0, EXPR_NUM,
3101
+ :tCOMMA, ",", EXPR_PAR,
3102
+ :tIDENTIFIER, "s", EXPR_ARG,
3103
+ :tEQL, "=", EXPR_BEG,
3104
+ :tFLOAT, 0.0, EXPR_NUM)
2741
3105
  end
2742
3106
 
2743
3107
  def test_zbug_id_equals
2744
3108
  assert_lex3("a = 0.0",
2745
3109
  nil,
2746
- :tIDENTIFIER, "a", :expr_cmdarg,
2747
- :tEQL, "=", :expr_beg,
2748
- :tFLOAT, 0.0, :expr_end)
3110
+ :tIDENTIFIER, "a", EXPR_CMDARG,
3111
+ :tEQL, "=", EXPR_BEG,
3112
+ :tFLOAT, 0.0, EXPR_NUM)
2749
3113
  end
2750
3114
 
2751
3115
  def test_zbug_no_spaces_in_decl
2752
3116
  assert_lex3("def initialize(u=0.0,s=0.0",
2753
3117
  nil,
2754
- :kDEF, "def", :expr_fname,
2755
- :tIDENTIFIER, "initialize", :expr_endfn,
2756
- :tLPAREN2, "(", :expr_beg,
2757
- :tIDENTIFIER, "u", :expr_arg,
2758
- :tEQL, "=", :expr_beg,
2759
- :tFLOAT, 0.0, :expr_end,
2760
- :tCOMMA, ",", :expr_beg,
2761
- :tIDENTIFIER, "s", :expr_arg,
2762
- :tEQL, "=", :expr_beg,
2763
- :tFLOAT, 0.0, :expr_end)
2764
- end
2765
-
2766
- def test_pct_w_backslashes
2767
- ["\t", "\n", "\r", "\v", "\f"].each do |char|
2768
- next if !RubyLexer::HAS_ENC and char == "\v"
2769
-
2770
- assert_lex("%w[foo#{char}bar]",
2771
- s(:array, s(:str, "foo"), s(:str, "bar")),
2772
-
2773
- :tQWORDS_BEG, "%w[", :expr_beg, 0, 0,
2774
- :tSTRING_CONTENT, "foo", :expr_beg, 0, 0,
2775
- :tSPACE, nil, :expr_beg, 0, 0,
2776
- :tSTRING_CONTENT, "bar", :expr_beg, 0, 0,
2777
- :tSPACE, nil, :expr_beg, 0, 0,
2778
- :tSTRING_END, nil, :expr_end, 0, 0)
2779
- end
2780
- end
2781
-
2782
- def test_yylex_sym_quoted
2783
- assert_lex(":'a'",
2784
- s(:lit, :a),
2785
-
2786
- :tSYMBOL, "a", :expr_end, 0, 0)
2787
- end
2788
-
2789
- def test_yylex_hash_colon
2790
- assert_lex("{a:1}",
2791
- s(:hash, s(:lit, :a), s(:lit, 1)),
2792
-
2793
- :tLBRACE, "{", :expr_beg, 0, 1,
2794
- :tLABEL, "a", :expr_labelarg, 0, 1,
2795
- :tINTEGER, 1, :expr_end, 0, 1,
2796
- :tRCURLY, "}", :expr_endarg, 0, 0)
2797
- end
2798
-
2799
- def test_yylex_hash_colon_quoted_22
2800
- setup_lexer_class RubyParser::V22
2801
-
2802
- assert_lex("{'a':1}",
2803
- s(:hash, s(:lit, :a), s(:lit, 1)),
2804
-
2805
- :tLBRACE, "{", :expr_beg, 0, 1,
2806
- :tLABEL, "a", :expr_labelarg, 0, 1,
2807
- :tINTEGER, 1, :expr_end, 0, 1,
2808
- :tRCURLY, "}", :expr_endarg, 0, 0)
2809
- end
2810
-
2811
- def test_yylex_hash_colon_quoted_symbol_22
2812
- setup_lexer_class RubyParser::V22
2813
-
2814
- assert_lex("{'abc': :b}",
2815
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2816
-
2817
- :tLBRACE, "{", :expr_beg, 0, 1,
2818
- :tLABEL, "abc", :expr_labelarg, 0, 1,
2819
- :tSYMBOL, "b", :expr_end, 0, 1,
2820
- :tRCURLY, "}", :expr_endarg, 0, 0)
2821
- end
2822
-
2823
- def test_yylex_hash_colon_double_quoted_symbol_22
2824
- setup_lexer_class RubyParser::V22
2825
-
2826
- assert_lex('{"abc": :b}',
2827
- s(:hash, s(:lit, :abc), s(:lit, :b)),
2828
-
2829
- :tLBRACE, "{", :expr_beg, 0, 1,
2830
- :tLABEL, "abc", :expr_labelarg, 0, 1,
2831
- :tSYMBOL, "b", :expr_end, 0, 1,
2832
- :tRCURLY, "}", :expr_endarg, 0, 0)
2833
- end
2834
-
2835
- def test_yylex_required_kwarg_no_value_22
2836
- setup_lexer_class RubyParser::V22
2837
-
2838
- assert_lex3("def foo a:, b:\nend",
2839
- nil,
2840
- :kDEF, "def", :expr_fname,
2841
- :tIDENTIFIER, "foo", :expr_endfn,
2842
- :tLABEL, "a", :expr_labelarg,
2843
- :tCOMMA, ",", :expr_beg,
2844
- :tLABEL, "b", :expr_labelarg,
2845
- :tNL, nil, :expr_beg,
2846
- :kEND, "end", :expr_end)
2847
- end
2848
-
2849
- def test_yylex_hash_colon_double_quoted_with_escapes
2850
- setup_lexer_class RubyParser::V22
2851
-
2852
- assert_lex3("{\"s\\tr\\i\\ng\\\\foo\\'bar\":1}",
2853
- nil,
2854
-
2855
- :tLBRACE, "{", :expr_beg,
2856
- :tLABEL, "s\tr\i\ng\\foo'bar", :expr_labelarg,
2857
- :tINTEGER, 1, :expr_end,
2858
- :tRCURLY, "}", :expr_endarg)
2859
- end
2860
-
2861
- def test_yylex_hash_colon_quoted_with_escapes
2862
- setup_lexer_class RubyParser::V22
2863
-
2864
- assert_lex3("{'s\\tr\\i\\ng\\\\foo\\'bar':1}",
2865
- nil,
2866
-
2867
- :tLBRACE, "{", :expr_beg,
2868
- :tLABEL, "s\\tr\\i\\ng\\foo'bar", :expr_labelarg,
2869
- :tINTEGER, 1, :expr_end,
2870
- :tRCURLY, "}", :expr_endarg)
2871
- end
2872
-
2873
- def test_ruby21_rational_literal
2874
- setup_lexer_class RubyParser::V21
2875
-
2876
- assert_lex3("10r", nil, :tRATIONAL, Rational(10), :expr_end)
2877
- assert_lex3("0x10r", nil, :tRATIONAL, Rational(16), :expr_end)
2878
- assert_lex3("0o10r", nil, :tRATIONAL, Rational(8), :expr_end)
2879
- assert_lex3("0or", nil, :tRATIONAL, Rational(0), :expr_end)
2880
- assert_lex3("0b10r", nil, :tRATIONAL, Rational(2), :expr_end)
2881
- assert_lex3("1.5r", nil, :tRATIONAL, Rational(15, 10), :expr_end)
2882
- assert_lex3("15e3r", nil, :tRATIONAL, Rational(15000), :expr_end)
2883
- assert_lex3("15e-3r", nil, :tRATIONAL, Rational(15, 1000), :expr_end)
2884
- assert_lex3("1.5e3r", nil, :tRATIONAL, Rational(1500), :expr_end)
2885
- assert_lex3("1.5e-3r", nil, :tRATIONAL, Rational(15, 10000), :expr_end)
2886
-
2887
- assert_lex3("-10r", nil,
2888
- :tUMINUS_NUM, "-", :expr_beg,
2889
- :tRATIONAL, Rational(10), :expr_end)
2890
- end
2891
-
2892
- def test_ruby21_imaginary_literal
2893
- setup_lexer_class RubyParser::V21
2894
-
2895
- assert_lex3("1i", nil, :tIMAGINARY, Complex(0, 1), :expr_end)
2896
- assert_lex3("0x10i", nil, :tIMAGINARY, Complex(0, 16), :expr_end)
2897
- assert_lex3("0o10i", nil, :tIMAGINARY, Complex(0, 8), :expr_end)
2898
- assert_lex3("0oi", nil, :tIMAGINARY, Complex(0, 0), :expr_end)
2899
- assert_lex3("0b10i", nil, :tIMAGINARY, Complex(0, 2), :expr_end)
2900
- assert_lex3("1.5i", nil, :tIMAGINARY, Complex(0, 1.5), :expr_end)
2901
- assert_lex3("15e3i", nil, :tIMAGINARY, Complex(0, 15000), :expr_end)
2902
- assert_lex3("15e-3i", nil, :tIMAGINARY, Complex(0, 0.015), :expr_end)
2903
- assert_lex3("1.5e3i", nil, :tIMAGINARY, Complex(0, 1500), :expr_end)
2904
- assert_lex3("1.5e-3i", nil, :tIMAGINARY, Complex(0, 0.0015), :expr_end)
2905
-
2906
- assert_lex3("-10i", nil,
2907
- :tUMINUS_NUM, "-", :expr_beg,
2908
- :tIMAGINARY, Complex(0, 10), :expr_end)
2909
- end
2910
-
2911
- def test_ruby21_rational_imaginary_literal
2912
- setup_lexer_class RubyParser::V21
2913
-
2914
- assert_lex3("1ri", nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
2915
- assert_lex3("0x10ri", nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
2916
- assert_lex3("0o10ri", nil, :tIMAGINARY, Complex(0, Rational(8)), :expr_end)
2917
- assert_lex3("0ori", nil, :tIMAGINARY, Complex(0, Rational(0)), :expr_end)
2918
- assert_lex3("0b10ri", nil, :tIMAGINARY, Complex(0, Rational(2)), :expr_end)
2919
- assert_lex3("1.5ri", nil, :tIMAGINARY, Complex(0, Rational("1.5")), :expr_end)
2920
- assert_lex3("15e3ri", nil, :tIMAGINARY, Complex(0, Rational("15e3")), :expr_end)
2921
- assert_lex3("15e-3ri", nil, :tIMAGINARY, Complex(0, Rational("15e-3")), :expr_end)
2922
- assert_lex3("1.5e3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e3")), :expr_end)
2923
- assert_lex3("1.5e-3ri", nil, :tIMAGINARY, Complex(0, Rational("1.5e-3")), :expr_end)
2924
-
2925
- assert_lex3("-10ri", nil,
2926
- :tUMINUS_NUM, "-", :expr_beg,
2927
- :tIMAGINARY, Complex(0, Rational(10)), :expr_end)
2928
- end
2929
-
2930
- def test_ruby21_imaginary_literal_with_succeeding_keyword
2931
- skip "Currently does not tokenize correctly"
2932
-
2933
- setup_lexer_class RubyParser::V21
2934
-
2935
- assert_lex3("1if", nil,
2936
- :tINTEGER, 1, :expr_end,
2937
- :kIF_MOD, "if", :expr_beg)
2938
- assert_lex3("1rif", nil,
2939
- :tRATIONAL, Rational(1), :expr_end,
2940
- :kIF_MOD, "if", :expr_beg)
2941
- assert_lex3("1.0if", nil,
2942
- :tFLOAT, 1.0, :expr_end,
2943
- :kIF_MOD, "if", :expr_beg)
2944
- assert_lex3("1.0rif", nil,
2945
- :tRATIONAL, Rational("1.0"), :expr_end,
2946
- :kIF_MOD, "if", :expr_beg)
2947
-
2948
- flunk
3118
+ :kDEF, "def", EXPR_FNAME,
3119
+ :tIDENTIFIER, "initialize", EXPR_ENDFN,
3120
+ :tLPAREN2, "(", EXPR_PAR,
3121
+ :tIDENTIFIER, "u", EXPR_ARG,
3122
+ :tEQL, "=", EXPR_BEG,
3123
+ :tFLOAT, 0.0, EXPR_NUM,
3124
+ :tCOMMA, ",", EXPR_PAR,
3125
+ :tIDENTIFIER, "s", EXPR_ARG,
3126
+ :tEQL, "=", EXPR_BEG,
3127
+ :tFLOAT, 0.0, EXPR_NUM)
2949
3128
  end
2950
3129
  end