ruby_parser 3.13.1 → 3.15.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.autotest +18 -29
  5. data/History.rdoc +97 -0
  6. data/Manifest.txt +2 -0
  7. data/README.rdoc +3 -3
  8. data/Rakefile +39 -21
  9. data/bin/ruby_parse +3 -1
  10. data/bin/ruby_parse_extract_error +8 -3
  11. data/compare/normalize.rb +43 -3
  12. data/debugging.md +39 -0
  13. data/lib/ruby20_parser.rb +3431 -3238
  14. data/lib/ruby20_parser.y +474 -300
  15. data/lib/ruby21_parser.rb +3491 -3295
  16. data/lib/ruby21_parser.y +482 -307
  17. data/lib/ruby22_parser.rb +3432 -3240
  18. data/lib/ruby22_parser.y +483 -308
  19. data/lib/ruby23_parser.rb +3338 -3150
  20. data/lib/ruby23_parser.y +483 -308
  21. data/lib/ruby24_parser.rb +3483 -3273
  22. data/lib/ruby24_parser.y +487 -310
  23. data/lib/ruby25_parser.rb +3482 -3272
  24. data/lib/ruby25_parser.y +487 -310
  25. data/lib/ruby26_parser.rb +3490 -3278
  26. data/lib/ruby26_parser.y +493 -314
  27. data/lib/ruby27_parser.rb +7224 -0
  28. data/lib/ruby27_parser.y +2657 -0
  29. data/lib/ruby_lexer.rb +483 -459
  30. data/lib/ruby_lexer.rex +5 -6
  31. data/lib/ruby_lexer.rex.rb +6 -8
  32. data/lib/ruby_parser.rb +29 -27
  33. data/lib/ruby_parser.yy +497 -315
  34. data/lib/ruby_parser_extras.rb +670 -420
  35. data/test/test_ruby_lexer.rb +1208 -1121
  36. data/test/test_ruby_parser.rb +2517 -1955
  37. data/test/test_ruby_parser_extras.rb +39 -4
  38. data/tools/munge.rb +10 -5
  39. data/tools/ripper.rb +13 -2
  40. metadata +28 -21
  41. metadata.gz.sig +0 -0
data/lib/ruby24_parser.y CHANGED
@@ -23,30 +23,30 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
23
23
  tLABEL_END
24
24
  tLONELY
25
25
 
26
- prechigh
27
- right tBANG tTILDE tUPLUS
28
- right tPOW
29
- right tUMINUS_NUM tUMINUS
30
- left tSTAR2 tDIVIDE tPERCENT
31
- left tPLUS tMINUS
32
- left tLSHFT tRSHFT
33
- left tAMPER2
34
- left tPIPE tCARET
35
- left tGT tGEQ tLT tLEQ
36
- nonassoc tCMP tEQ tEQQ tNEQ tMATCH tNMATCH
37
- left tANDOP
38
- left tOROP
39
- nonassoc tDOT2 tDOT3
40
- right tEH tCOLON
41
- left kRESCUE_MOD
42
- right tEQL tOP_ASGN
43
- nonassoc kDEFINED
44
- right kNOT
45
- left kOR kAND
46
- nonassoc kIF_MOD kUNLESS_MOD kWHILE_MOD kUNTIL_MOD
47
- nonassoc tLBRACE_ARG
48
- nonassoc tLOWEST
49
26
  preclow
27
+ nonassoc tLOWEST
28
+ nonassoc tLBRACE_ARG
29
+ nonassoc kIF_MOD kUNLESS_MOD kWHILE_MOD kUNTIL_MOD
30
+ left kOR kAND
31
+ right kNOT
32
+ nonassoc kDEFINED
33
+ right tEQL tOP_ASGN
34
+ left kRESCUE_MOD
35
+ right tEH tCOLON
36
+ nonassoc tDOT2 tDOT3
37
+ left tOROP
38
+ left tANDOP
39
+ nonassoc tCMP tEQ tEQQ tNEQ tMATCH tNMATCH
40
+ left tGT tGEQ tLT tLEQ
41
+ left tPIPE tCARET
42
+ left tAMPER2
43
+ left tLSHFT tRSHFT
44
+ left tPLUS tMINUS
45
+ left tSTAR2 tDIVIDE tPERCENT # TODO: tSTAR2 -> tMULT
46
+ right tUMINUS_NUM tUMINUS
47
+ right tPOW
48
+ right tBANG tTILDE tUPLUS
49
+ prechigh
50
50
 
51
51
  rule
52
52
 
@@ -60,7 +60,8 @@ rule
60
60
 
61
61
  top_compstmt: top_stmts opt_terms
62
62
  {
63
- result = val[0]
63
+ stmt, _ = val
64
+ result = stmt
64
65
  }
65
66
 
66
67
  top_stmts: none
@@ -72,14 +73,6 @@ rule
72
73
  | error top_stmt
73
74
 
74
75
  top_stmt: stmt
75
- {
76
- result = val[0]
77
-
78
- # TODO: remove once I have more confidence this is fixed
79
- # result.each_of_type :call_args do |s|
80
- # debug20 666, s, result
81
- # end
82
- }
83
76
  | klBEGIN
84
77
  {
85
78
  if (self.in_def || self.in_single > 0) then
@@ -90,14 +83,19 @@ rule
90
83
  }
91
84
  begin_block
92
85
  {
93
- _, _, block = val
94
- result = block
86
+ (_, lineno), _, iter = val
87
+ iter.line lineno
88
+
89
+ (_, preexe,) = iter
90
+ preexe.line lineno
91
+
92
+ result = iter
95
93
  }
96
94
 
97
- begin_block: tLCURLY top_compstmt tRCURLY
95
+ begin_block: tLCURLY { result = lexer.lineno } top_compstmt tRCURLY
98
96
  {
99
- _, stmt, _ = val
100
- result = new_iter s(:preexe), 0, stmt
97
+ _, line, stmt, _ = val
98
+ result = new_iter s(:preexe).line(line), 0, stmt
101
99
  }
102
100
 
103
101
  bodystmt: compstmt opt_rescue k_else
@@ -139,34 +137,27 @@ rule
139
137
  stmt_or_begin: stmt
140
138
  | klBEGIN
141
139
  {
142
- if (self.in_def || self.in_single > 0) then
143
- debug20 1
144
- yyerror "BEGIN in method"
145
- end
146
- self.env.extend
147
- }
148
- begin_block
149
- {
150
- _, _, stmt = val
151
- result = stmt
140
+ yyerror "BEGIN is permitted only at toplevel"
152
141
  }
153
142
 
154
143
  stmt: kALIAS fitem
155
144
  {
156
145
  lexer.lex_state = EXPR_FNAME
157
- result = self.lexer.lineno
158
146
  }
159
147
  fitem
160
148
  {
161
- result = s(:alias, val[1], val[3]).line(val[2])
149
+ (_, line), lhs, _, rhs = val
150
+ result = s(:alias, lhs, rhs).line(line).line line
162
151
  }
163
152
  | kALIAS tGVAR tGVAR
164
153
  {
165
- result = s(:valias, val[1].to_sym, val[2].to_sym)
154
+ (_, line), lhs, rhs = val
155
+ result = s(:valias, lhs.to_sym, rhs.to_sym).line line
166
156
  }
167
157
  | kALIAS tGVAR tBACK_REF
168
158
  {
169
- result = s(:valias, val[1].to_sym, :"$#{val[2]}")
159
+ (_, line), lhs, rhs = val
160
+ result = s(:valias, lhs.to_sym, :"$#{rhs}").line line
170
161
  }
171
162
  | kALIAS tGVAR tNTH_REF
172
163
  {
@@ -178,32 +169,41 @@ rule
178
169
  }
179
170
  | stmt kIF_MOD expr_value
180
171
  {
181
- result = new_if val[2], val[0], nil
172
+ t, _, c = val
173
+ result = new_if c, t, nil
182
174
  }
183
175
  | stmt kUNLESS_MOD expr_value
184
176
  {
185
- result = new_if val[2], nil, val[0]
177
+ f, _, c = val
178
+ result = new_if c, nil, f
186
179
  }
187
180
  | stmt kWHILE_MOD expr_value
188
181
  {
189
- result = new_while val[0], val[2], true
182
+ e, _, c = val
183
+ result = new_while e, c, true
190
184
  }
191
185
  | stmt kUNTIL_MOD expr_value
192
186
  {
193
- result = new_until val[0], val[2], true
187
+ e, _, c = val
188
+ result = new_until e, c, true
194
189
  }
195
190
  | stmt kRESCUE_MOD stmt
196
191
  {
197
192
  body, _, resbody = val
198
- result = new_rescue body, new_resbody(s(:array), resbody)
193
+
194
+ resbody = new_resbody s(:array).line(resbody.line), resbody
195
+ result = new_rescue body, resbody
199
196
  }
200
197
  | klEND tLCURLY compstmt tRCURLY
201
198
  {
199
+ (_, line), _, stmt, _ = val
200
+
202
201
  if (self.in_def || self.in_single > 0) then
203
202
  debug20 3
204
203
  yyerror "END in method; use at_exit"
205
204
  end
206
- result = new_iter s(:postexe), 0, val[2]
205
+
206
+ result = new_iter s(:postexe).line(line), 0, stmt
207
207
  }
208
208
  | command_asgn
209
209
  | mlhs tEQL command_call
@@ -212,7 +212,8 @@ rule
212
212
  }
213
213
  | lhs tEQL mrhs
214
214
  {
215
- result = new_assign val[0], s(:svalue, val[2])
215
+ lhs, _, rhs = val
216
+ result = new_assign lhs, s(:svalue, rhs).line(rhs.line)
216
217
  }
217
218
  | mlhs tEQL mrhs_arg
218
219
  {
@@ -234,11 +235,12 @@ rule
234
235
  }
235
236
  | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN command_rhs
236
237
  {
237
- result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
238
+ result = new_op_asgn1 val
238
239
  }
239
240
  | primary_value call_op tIDENTIFIER tOP_ASGN command_rhs
240
241
  {
241
- result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
242
+ prim, _, id, opasgn, rhs = val
243
+ result = s(:op_asgn, prim, rhs, id.to_sym, opasgn.to_sym)
242
244
  if val[1] == '&.'
243
245
  result.sexp_type = :safe_op_asgn
244
246
  end
@@ -254,13 +256,15 @@ rule
254
256
  }
255
257
  | primary_value tCOLON2 tCONSTANT tOP_ASGN command_rhs
256
258
  {
257
- result = s(:op_asgn, val[0], val[4], val[2], val[3])
258
- debug20 4, val, result
259
+ lhs1, _, lhs2, op, rhs = val
260
+
261
+ result = s(:op_asgn, lhs1, rhs, lhs2.to_sym, op.to_sym)
259
262
  }
260
263
  | primary_value tCOLON2 tIDENTIFIER tOP_ASGN command_rhs
261
264
  {
262
- result = s(:op_asgn, val[0], val[4], val[2], val[3])
263
- debug20 5, val, result
265
+ lhs1, _, lhs2, op, rhs = val
266
+
267
+ result = s(:op_asgn, lhs1, rhs, lhs2.to_sym, op.to_sym)
264
268
  }
265
269
  | backref tOP_ASGN command_rhs
266
270
  {
@@ -274,28 +278,37 @@ rule
274
278
  }
275
279
  | command_call kRESCUE_MOD stmt
276
280
  {
277
- expr, _, resbody = val
281
+ expr, (_, line), resbody = val
282
+
278
283
  expr = value_expr expr
279
- result = new_rescue(expr, new_resbody(s(:array), resbody))
284
+ ary = s(:array).line line
285
+ result = new_rescue(expr, new_resbody(ary, resbody))
280
286
  }
281
287
  | command_asgn
282
288
 
283
289
  expr: command_call
284
290
  | expr kAND expr
285
291
  {
286
- result = logical_op :and, val[0], val[2]
292
+ lhs, _, rhs = val
293
+ result = logical_op :and, lhs, rhs
287
294
  }
288
295
  | expr kOR expr
289
296
  {
290
- result = logical_op :or, val[0], val[2]
297
+ lhs, _, rhs = val
298
+ result = logical_op :or, lhs, rhs
291
299
  }
292
300
  | kNOT opt_nl expr
293
301
  {
294
- result = s(:call, val[2], :"!")
302
+ (_, line), _, expr = val
303
+ result = new_call(expr, :"!").line line
304
+ # REFACTOR: call_uni_op
295
305
  }
296
306
  | tBANG command_call
297
307
  {
298
- result = s(:call, val[1], :"!")
308
+ _, cmd = val
309
+ result = new_call(cmd, :"!").line cmd.line
310
+ # TODO: fix line number to tBANG... but causes BAD shift/reduce conflict
311
+ # REFACTOR: call_uni_op -- see parse26.y
299
312
  }
300
313
  | arg
301
314
 
@@ -322,7 +335,8 @@ rule
322
335
  block_command: block_call
323
336
  | block_call call_op2 operation2 command_args
324
337
  {
325
- result = new_call val[0], val[2].to_sym, val[3]
338
+ blk, _, msg, args = val
339
+ result = new_call(blk, msg.to_sym, args).line blk.line
326
340
  }
327
341
 
328
342
  cmd_brace_block: tLBRACE_ARG
@@ -342,26 +356,32 @@ rule
342
356
 
343
357
  fcall: operation
344
358
  {
345
- result = new_call nil, val[0].to_sym
359
+ msg, = val
360
+ result = new_call(nil, msg.to_sym).line lexer.lineno
346
361
  }
347
362
 
348
363
  command: fcall command_args =tLOWEST
349
364
  {
350
- result = val[0].concat val[1].sexp_body # REFACTOR pattern
365
+ call, args = val
366
+ result = call.concat args.sexp_body
351
367
  }
352
368
  | fcall command_args cmd_brace_block
353
369
  {
354
- result = val[0].concat val[1].sexp_body
355
- if val[2] then
356
- block_dup_check result, val[2]
370
+ call, args, block = val
357
371
 
358
- result, operation = val[2], result
372
+ result = call.concat args.sexp_body
373
+
374
+ if block then
375
+ block_dup_check result, block
376
+
377
+ result, operation = block, result
359
378
  result.insert 1, operation
360
379
  end
361
380
  }
362
381
  | primary_value call_op operation2 command_args =tLOWEST
363
382
  {
364
- result = new_call val[0], val[2].to_sym, val[3], val[1]
383
+ lhs, callop, op, args = val
384
+ result = new_call lhs, op.to_sym, args, callop
365
385
  }
366
386
  | primary_value call_op operation2 command_args cmd_brace_block
367
387
  {
@@ -393,7 +413,9 @@ rule
393
413
  }
394
414
  | kYIELD command_args
395
415
  {
396
- result = new_yield val[1]
416
+ (_, line), args = val
417
+ result = new_yield args
418
+ result.line line # TODO: push to new_yield
397
419
  }
398
420
  | k_return call_args
399
421
  {
@@ -402,8 +424,8 @@ rule
402
424
  }
403
425
  | kBREAK call_args
404
426
  {
405
- line = val[0].last
406
- result = s(:break, ret_args(val[1])).line(line)
427
+ (_, line), args = val
428
+ result = s(:break, ret_args(args)).line line
407
429
  }
408
430
  | kNEXT call_args
409
431
  {
@@ -420,56 +442,79 @@ rule
420
442
  mlhs_inner: mlhs_basic
421
443
  | tLPAREN mlhs_inner rparen
422
444
  {
423
- result = s(:masgn, s(:array, val[1]))
445
+ _, arg, _ = val
446
+ l = arg.line
447
+
448
+ result = s(:masgn, s(:array, arg).line(l)).line l
424
449
  }
425
450
 
426
451
  mlhs_basic: mlhs_head
427
452
  {
428
- result = s(:masgn, val[0])
453
+ head, = val
454
+ result = s(:masgn, head).line head.line
429
455
  }
430
456
  | mlhs_head mlhs_item
431
457
  {
432
- result = s(:masgn, val[0] << val[1].compact)
458
+ lhs, rhs = val
459
+ result = s(:masgn, lhs << rhs.compact).line lhs.line
433
460
  }
434
461
  | mlhs_head tSTAR mlhs_node
435
462
  {
436
- result = s(:masgn, val[0] << s(:splat, val[2]))
463
+ head, _, tail = val
464
+ head << s(:splat, tail).line(tail.line)
465
+ result = s(:masgn, head).line head.line
437
466
  }
438
467
  | mlhs_head tSTAR mlhs_node tCOMMA mlhs_post
439
468
  {
440
469
  ary1, _, splat, _, ary2 = val
441
470
 
442
- result = list_append ary1, s(:splat, splat)
471
+ result = list_append ary1, s(:splat, splat).line(splat.line)
443
472
  result.concat ary2.sexp_body
444
- result = s(:masgn, result)
473
+ result = s(:masgn, result).line result.line
445
474
  }
446
475
  | mlhs_head tSTAR
447
476
  {
448
- result = s(:masgn, val[0] << s(:splat))
477
+ head, _ = val
478
+ l = head.line
479
+ result = s(:masgn, head << s(:splat).line(l)).line l
449
480
  }
450
481
  | mlhs_head tSTAR tCOMMA mlhs_post
451
482
  {
452
- ary = list_append val[0], s(:splat)
453
- ary.concat val[3].sexp_body
454
- result = s(:masgn, ary)
483
+ head, _, _, post = val
484
+ ary = list_append head, s(:splat).line(head.line)
485
+ ary.concat post.sexp_body
486
+ result = s(:masgn, ary).line ary.line
455
487
  }
456
488
  | tSTAR mlhs_node
457
489
  {
458
- result = s(:masgn, s(:array, s(:splat, val[1])))
490
+ _, node = val
491
+ l = node.line
492
+ splat = s(:splat, node).line l
493
+ ary = s(:array, splat).line l
494
+ result = s(:masgn, ary).line l
459
495
  }
460
496
  | tSTAR mlhs_node tCOMMA mlhs_post
461
497
  {
462
- ary = s(:array, s(:splat, val[1]))
463
- ary.concat val[3].sexp_body
464
- result = s(:masgn, ary)
498
+ _, node, _, post = val
499
+
500
+ splat = s(:splat, node).line node.line
501
+ ary = s(:array, splat).line splat.line
502
+ ary.concat post.sexp_body
503
+ result = s(:masgn, ary).line ary.line
465
504
  }
466
505
  | tSTAR
467
506
  {
468
- result = s(:masgn, s(:array, s(:splat)))
507
+ l = lexer.lineno
508
+ result = s(:masgn, s(:array, s(:splat).line(l)).line(l)).line l
469
509
  }
470
510
  | tSTAR tCOMMA mlhs_post
471
511
  {
472
- result = s(:masgn, s(:array, s(:splat), *val[2].sexp_body))
512
+ _, _, post = val
513
+ l = post.line
514
+
515
+ splat = s(:splat).line l
516
+ ary = s(:array, splat, *post.sexp_body).line l
517
+ result = s(:masgn, ary).line l
473
518
  }
474
519
 
475
520
  mlhs_item: mlhs_node
@@ -480,7 +525,8 @@ rule
480
525
 
481
526
  mlhs_head: mlhs_item tCOMMA
482
527
  {
483
- result = s(:array, val[0])
528
+ lhs, _ = val
529
+ result = s(:array, lhs).line lhs.line
484
530
  }
485
531
  | mlhs_head mlhs_item tCOMMA
486
532
  {
@@ -489,7 +535,8 @@ rule
489
535
 
490
536
  mlhs_post: mlhs_item
491
537
  {
492
- result = s(:array, val[0])
538
+ item, = val
539
+ result = s(:array, item).line item.line
493
540
  }
494
541
  | mlhs_post tCOMMA mlhs_item
495
542
  {
@@ -514,7 +561,8 @@ rule
514
561
  }
515
562
  | primary_value tCOLON2 tIDENTIFIER
516
563
  {
517
- result = s(:attrasgn, val[0], :"#{val[2]}=")
564
+ recv, _, id = val
565
+ result = new_attrasgn recv, id
518
566
  }
519
567
  | primary_value call_op tCONSTANT
520
568
  {
@@ -527,7 +575,10 @@ rule
527
575
  yyerror "dynamic constant assignment"
528
576
  end
529
577
 
530
- result = s(:const, s(:colon2, val[0], val[2].to_sym), nil)
578
+ expr, _, id = val
579
+ l = expr.line
580
+
581
+ result = s(:const, s(:colon2, expr, id.to_sym).line(l), nil).line l
531
582
  }
532
583
  | tCOLON3 tCONSTANT
533
584
  {
@@ -536,7 +587,10 @@ rule
536
587
  yyerror "dynamic constant assignment"
537
588
  end
538
589
 
539
- result = s(:const, nil, s(:colon3, val[1].to_sym))
590
+ _, id = val
591
+ l = lexer.lineno
592
+
593
+ result = s(:const, nil, s(:colon3, id.to_sym).line(l)).line l
540
594
  }
541
595
  | backref
542
596
  {
@@ -545,24 +599,31 @@ rule
545
599
 
546
600
  lhs: user_variable
547
601
  {
602
+ line = lexer.lineno
548
603
  result = self.assignable val[0]
604
+ result.line = line
549
605
  }
550
606
  | keyword_variable
551
607
  {
608
+ line = lexer.lineno
552
609
  result = self.assignable val[0]
610
+ result.line = line
553
611
  debug20 9, val, result
554
612
  }
555
613
  | primary_value tLBRACK2 opt_call_args rbracket
556
614
  {
557
- result = self.aryset val[0], val[2]
615
+ lhs, _, args, _ = val
616
+ result = self.aryset lhs, args
558
617
  }
559
618
  | primary_value call_op tIDENTIFIER # REFACTOR
560
619
  {
561
- result = new_attrasgn val[0], val[2], val[1]
620
+ lhs, op, id = val
621
+ result = new_attrasgn lhs, id, op
562
622
  }
563
623
  | primary_value tCOLON2 tIDENTIFIER
564
624
  {
565
- result = s(:attrasgn, val[0], :"#{val[2]}=")
625
+ lhs, _, id = val
626
+ result = new_attrasgn lhs, id
566
627
  }
567
628
  | primary_value call_op tCONSTANT # REFACTOR?
568
629
  {
@@ -570,21 +631,27 @@ rule
570
631
  }
571
632
  | primary_value tCOLON2 tCONSTANT
572
633
  {
634
+ expr, _, id = val
635
+
573
636
  if (self.in_def || self.in_single > 0) then
574
637
  debug20 10
575
638
  yyerror "dynamic constant assignment"
576
639
  end
577
640
 
578
- result = s(:const, s(:colon2, val[0], val[2].to_sym))
641
+ l = expr.line
642
+ result = s(:const, s(:colon2, expr, id.to_sym).line(l)).line l
579
643
  }
580
644
  | tCOLON3 tCONSTANT
581
645
  {
646
+ _, id = val
647
+
582
648
  if (self.in_def || self.in_single > 0) then
583
649
  debug20 11
584
650
  yyerror "dynamic constant assignment"
585
651
  end
586
652
 
587
- result = s(:const, s(:colon3, val[1].to_sym))
653
+ l = lexer.lineno
654
+ result = s(:const, s(:colon3, id.to_sym).line(l)).line l
588
655
  }
589
656
  | backref
590
657
  {
@@ -599,7 +666,8 @@ rule
599
666
 
600
667
  cpath: tCOLON3 cname
601
668
  {
602
- result = s(:colon3, val[1].to_sym)
669
+ _, name = val
670
+ result = s(:colon3, name.to_sym).line lexer.lineno
603
671
  }
604
672
  | cname
605
673
  {
@@ -607,7 +675,10 @@ rule
607
675
  }
608
676
  | primary_value tCOLON2 cname
609
677
  {
610
- result = s(:colon2, val[0], val[2].to_sym)
678
+ pval, _, name = val
679
+
680
+ result = s(:colon2, pval, name.to_sym)
681
+ result.line pval.line
611
682
  }
612
683
 
613
684
  fname: tIDENTIFIER | tCONSTANT | tFID
@@ -628,7 +699,8 @@ rule
628
699
 
629
700
  fitem: fsym
630
701
  {
631
- result = s(:lit, val[0].to_sym)
702
+ id, = val
703
+ result = s(:lit, id.to_sym).line lexer.lineno
632
704
  }
633
705
  | dsym
634
706
 
@@ -651,6 +723,7 @@ rule
651
723
  | tNEQ | tLSHFT | tRSHFT | tPLUS | tMINUS | tSTAR2
652
724
  | tSTAR | tDIVIDE | tPERCENT | tPOW | tDSTAR | tBANG | tTILDE
653
725
  | tUPLUS | tUMINUS | tAREF | tASET | tBACK_REF2
726
+ # TODO: tUBANG dead?
654
727
  | tUBANG
655
728
 
656
729
  reswords: k__LINE__ | k__FILE__ | k__ENCODING__ | klBEGIN | klEND
@@ -673,8 +746,7 @@ rule
673
746
  }
674
747
  | primary_value tLBRACK2 opt_call_args rbracket tOP_ASGN arg_rhs
675
748
  {
676
- val[2].sexp_type = :arglist if val[2]
677
- result = s(:op_asgn1, val[0], val[2], val[4].to_sym, val[5])
749
+ result = new_op_asgn1 val
678
750
  }
679
751
  | primary_value call_op tIDENTIFIER tOP_ASGN arg_rhs
680
752
  {
@@ -686,7 +758,9 @@ rule
686
758
  }
687
759
  | primary_value tCOLON2 tIDENTIFIER tOP_ASGN arg_rhs
688
760
  {
689
- result = s(:op_asgn, val[0], val[4], val[2].to_sym, val[3].to_sym)
761
+ lhs, _, id, op, rhs = val
762
+
763
+ result = s(:op_asgn, lhs, rhs, id.to_sym, op.to_sym).line lhs.line
690
764
  }
691
765
  | primary_value tCOLON2 tCONSTANT tOP_ASGN arg_rhs
692
766
  {
@@ -716,18 +790,18 @@ rule
716
790
  {
717
791
  v1, v2 = val[0], val[2]
718
792
  if v1.node_type == :lit and v2.node_type == :lit and Integer === v1.last and Integer === v2.last then
719
- result = s(:lit, (v1.last)..(v2.last))
793
+ result = s(:lit, (v1.last)..(v2.last)).line v1.line
720
794
  else
721
- result = s(:dot2, v1, v2)
795
+ result = s(:dot2, v1, v2).line v1.line
722
796
  end
723
797
  }
724
798
  | arg tDOT3 arg
725
799
  {
726
800
  v1, v2 = val[0], val[2]
727
801
  if v1.node_type == :lit and v2.node_type == :lit and Integer === v1.last and Integer === v2.last then
728
- result = s(:lit, (v1.last)...(v2.last))
802
+ result = s(:lit, (v1.last)...(v2.last)).line v1.line
729
803
  else
730
- result = s(:dot3, v1, v2)
804
+ result = s(:dot3, v1, v2).line v1.line
731
805
  end
732
806
  }
733
807
  | arg tPLUS arg
@@ -756,7 +830,9 @@ rule
756
830
  }
757
831
  | tUMINUS_NUM simple_numeric tPOW arg
758
832
  {
759
- result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
833
+ lit = s(:lit, val[1]).line lexer.lineno
834
+ result = new_call(new_call(lit, :"**", argl(val[3])), :"-@")
835
+
760
836
  }
761
837
  | tUPLUS arg
762
838
  {
@@ -797,15 +873,19 @@ rule
797
873
  }
798
874
  | arg tMATCH arg
799
875
  {
800
- result = new_match val[0], val[2]
876
+ lhs, _, rhs = val
877
+ result = new_match lhs, rhs
801
878
  }
802
879
  | arg tNMATCH arg
803
880
  {
804
- result = s(:not, new_match(val[0], val[2]))
881
+ lhs, _, rhs = val
882
+ result = s(:not, new_match(lhs, rhs)).line lhs.line
805
883
  }
806
884
  | tBANG arg
807
885
  {
808
- result = new_call val[1], :"!"
886
+ _, arg = val
887
+ result = new_call arg, :"!"
888
+ result.line arg.line
809
889
  }
810
890
  | tTILDE arg
811
891
  {
@@ -833,11 +913,13 @@ rule
833
913
  }
834
914
  | kDEFINED opt_nl arg
835
915
  {
836
- result = s(:defined, val[2])
916
+ (_, line), _, arg = val
917
+ result = s(:defined, arg).line line
837
918
  }
838
919
  | arg tEH arg opt_nl tCOLON arg
839
920
  {
840
- result = s(:if, val[0], val[2], val[5])
921
+ c, _, t, _, _, f = val
922
+ result = s(:if, c, t, f).line c.line
841
923
  }
842
924
  | primary
843
925
 
@@ -880,28 +962,25 @@ rule
880
962
  arg_rhs: arg =tOP_ASGN
881
963
  | arg kRESCUE_MOD arg
882
964
  {
883
- body, _, resbody = val
965
+ body, (_, line), resbody = val
884
966
  body = value_expr body
885
967
  resbody = remove_begin resbody
886
- result = new_rescue(body, new_resbody(s(:array), resbody))
968
+
969
+ ary = s(:array).line line
970
+ result = new_rescue(body, new_resbody(ary, resbody))
887
971
  }
888
972
 
889
973
  paren_args: tLPAREN2 opt_call_args rparen
890
974
  {
891
- result = val[1]
975
+ _, args, _ = val
976
+ result = args
892
977
  }
893
978
 
894
979
  opt_paren_args: none
895
980
  | paren_args
896
981
 
897
982
  opt_call_args: none
898
- {
899
- result = val[0]
900
- }
901
983
  | call_args
902
- {
903
- result = val[0]
904
- }
905
984
  | args tCOMMA
906
985
  {
907
986
  result = args val
@@ -923,17 +1002,14 @@ rule
923
1002
  | args opt_block_arg
924
1003
  {
925
1004
  result = call_args val
926
- result = self.arg_blk_pass val[0], val[1]
927
1005
  }
928
1006
  | assocs opt_block_arg
929
1007
  {
930
- result = call_args [array_to_hash(val[0])]
931
- result = self.arg_blk_pass result, val[1]
1008
+ result = call_args [array_to_hash(val[0]), val[1]]
932
1009
  }
933
1010
  | args tCOMMA assocs opt_block_arg
934
1011
  {
935
- result = call_args [val[0], array_to_hash(val[2])]
936
- result = self.arg_blk_pass result, val[3]
1012
+ result = call_args [val[0], array_to_hash(val[2]), val[3]]
937
1013
  }
938
1014
  | block_arg
939
1015
  {
@@ -941,17 +1017,45 @@ rule
941
1017
  }
942
1018
 
943
1019
  command_args: {
944
- result = lexer.cmdarg.store true
1020
+ # parse26.y line 2200
1021
+
1022
+ # If call_args starts with a open paren '(' or
1023
+ # '[', look-ahead reading of the letters calls
1024
+ # CMDARG_PUSH(0), but the push must be done
1025
+ # after CMDARG_PUSH(1). So this code makes them
1026
+ # consistent by first cancelling the premature
1027
+ # CMDARG_PUSH(0), doing CMDARG_PUSH(1), and
1028
+ # finally redoing CMDARG_PUSH(0).
1029
+
1030
+ result = yychar = self.last_token_type.first
1031
+ lookahead = [:tLPAREN, :tLPAREN_ARG, :tLPAREN2, :tLBRACK, :tLBRACK2].include?(yychar)
1032
+ lexer.cmdarg.pop if lookahead
1033
+ lexer.cmdarg.push true
1034
+ lexer.cmdarg.push false if lookahead
945
1035
  }
946
1036
  call_args
947
1037
  {
948
- lexer.cmdarg.restore val[0]
949
- result = val[1]
1038
+ yychar, args = val
1039
+
1040
+ # call_args can be followed by tLBRACE_ARG (that
1041
+ # does CMDARG_PUSH(0) in the lexer) but the push
1042
+ # must be done after CMDARG_POP() in the parser.
1043
+ # So this code does CMDARG_POP() to pop 0 pushed
1044
+ # by tLBRACE_ARG, CMDARG_POP() to pop 1 pushed
1045
+ # by command_args, and CMDARG_PUSH(0) to restore
1046
+ # back the flag set by tLBRACE_ARG.
1047
+
1048
+ lookahead = [:tLBRACE_ARG].include?(yychar)
1049
+ lexer.cmdarg.pop if lookahead
1050
+ lexer.cmdarg.pop
1051
+ lexer.cmdarg.push false if lookahead
1052
+ result = args
950
1053
  }
951
1054
 
952
1055
  block_arg: tAMPER arg_value
953
1056
  {
954
- result = s(:block_pass, val[1])
1057
+ _, arg = val
1058
+ result = s(:block_pass, arg).line arg.line
955
1059
  }
956
1060
 
957
1061
  opt_block_arg: tCOMMA block_arg
@@ -962,19 +1066,27 @@ rule
962
1066
 
963
1067
  args: arg_value
964
1068
  {
965
- result = s(:array, val[0])
1069
+ arg, = val
1070
+ lineno = arg.line || lexer.lineno # HACK
1071
+
1072
+ result = s(:array, arg).line lineno
966
1073
  }
967
1074
  | tSTAR arg_value
968
1075
  {
969
- result = s(:array, s(:splat, val[1]))
1076
+ _, arg = val
1077
+ result = s(:array, s(:splat, arg).line(arg.line)).line arg.line
970
1078
  }
971
1079
  | args tCOMMA arg_value
972
1080
  {
973
- result = self.list_append val[0], val[2]
1081
+ args, _, id = val
1082
+ result = self.list_append args, id
974
1083
  }
975
1084
  | args tCOMMA tSTAR arg_value
976
1085
  {
977
- result = self.list_append val[0], s(:splat, val[3])
1086
+ # TODO: the line number from tSTAR has been dropped
1087
+ args, _, _, id = val
1088
+ line = lexer.lineno
1089
+ result = self.list_append args, s(:splat, id).line(line)
978
1090
  }
979
1091
 
980
1092
  mrhs_arg: mrhs
@@ -992,11 +1104,14 @@ rule
992
1104
  }
993
1105
  | args tCOMMA tSTAR arg_value
994
1106
  {
995
- result = self.arg_concat val[0], val[3]
1107
+ # TODO: make all tXXXX terminals include lexer.lineno
1108
+ arg, _, _, splat = val
1109
+ result = self.arg_concat arg, splat
996
1110
  }
997
1111
  | tSTAR arg_value
998
1112
  {
999
- result = s(:splat, val[1])
1113
+ _, arg = val
1114
+ result = s(:splat, arg).line arg.line
1000
1115
  }
1001
1116
 
1002
1117
  primary: literal
@@ -1011,65 +1126,65 @@ rule
1011
1126
  | backref
1012
1127
  | tFID
1013
1128
  {
1014
- result = new_call nil, val[0].to_sym
1129
+ msg, = val
1130
+ result = new_call nil, msg.to_sym
1015
1131
  }
1016
1132
  | k_begin
1017
1133
  {
1134
+ lexer.cmdarg.push false
1018
1135
  result = self.lexer.lineno
1019
- # TODO:
1020
- # $<val>1 = cmdarg_stack;
1021
- # CMDARG_SET(0);
1022
1136
  }
1023
1137
  bodystmt k_end
1024
1138
  {
1025
- # TODO: CMDARG_SET($<val>1);
1026
- unless val[2] then
1027
- result = s(:nil)
1028
- else
1029
- result = s(:begin, val[2])
1030
- end
1031
-
1032
- result.line = val[1]
1139
+ lexer.cmdarg.pop
1140
+ result = new_begin val
1033
1141
  }
1034
- | tLPAREN_ARG rparen
1142
+ | tLPAREN_ARG
1035
1143
  {
1036
- # TODO: lex_state = EXPR_ENDARG in between
1037
- debug20 13, val, result
1144
+ lexer.lex_state = EXPR_ENDARG
1145
+ result = lexer.lineno
1038
1146
  }
1039
- | tLPAREN_ARG
1147
+ rparen
1040
1148
  {
1041
- result = lexer.cmdarg.store false
1042
- # result = self.lexer.cmdarg.stack.dup
1043
- # lexer.cmdarg.stack.replace [false] # TODO add api for these
1149
+ _, line, _ = val
1150
+ result = s(:begin).line line
1044
1151
  }
1152
+ | tLPAREN_ARG
1045
1153
  stmt
1046
1154
  {
1047
1155
  lexer.lex_state = EXPR_ENDARG
1048
1156
  }
1049
1157
  rparen
1050
1158
  {
1051
- _, cmdarg, stmt, _, _, = val
1052
- warning "(...) interpreted as grouped expression"
1053
- lexer.cmdarg.restore cmdarg
1159
+ _, stmt, _, _, = val
1160
+ # warning "(...) interpreted as grouped expression"
1054
1161
  result = stmt
1055
1162
  }
1056
1163
  | tLPAREN compstmt tRPAREN
1057
1164
  {
1058
- result = val[1] || s(:nil)
1165
+ _, stmt, _ = val
1166
+ result = stmt
1167
+ result ||= s(:nil).line lexer.lineno
1059
1168
  result.paren = true
1060
1169
  }
1061
1170
  | primary_value tCOLON2 tCONSTANT
1062
1171
  {
1063
- result = s(:colon2, val[0], val[2].to_sym)
1172
+ expr, _, id = val
1173
+
1174
+ result = s(:colon2, expr, id.to_sym).line expr.line
1064
1175
  }
1065
1176
  | tCOLON3 tCONSTANT
1066
1177
  {
1067
- result = s(:colon3, val[1].to_sym)
1178
+ _, id = val
1179
+
1180
+ result = s(:colon3, id.to_sym).line lexer.lineno
1068
1181
  }
1069
- | tLBRACK aref_args tRBRACK
1182
+ | tLBRACK { result = lexer.lineno } aref_args tRBRACK
1070
1183
  {
1071
- result = val[1] || s(:array)
1184
+ _, line, args, _ = val
1185
+ result = args || s(:array)
1072
1186
  result.sexp_type = :array # aref_args is :args
1187
+ result.line line
1073
1188
  }
1074
1189
  | tLBRACE
1075
1190
  {
@@ -1081,7 +1196,8 @@ rule
1081
1196
  }
1082
1197
  | k_return
1083
1198
  {
1084
- result = s(:return)
1199
+ (_, line), = val
1200
+ result = s(:return).line line
1085
1201
  }
1086
1202
  | kYIELD tLPAREN2 call_args rparen
1087
1203
  {
@@ -1097,11 +1213,14 @@ rule
1097
1213
  }
1098
1214
  | kDEFINED opt_nl tLPAREN2 expr rparen
1099
1215
  {
1100
- result = s(:defined, val[3])
1216
+ (_, line), _, _, arg, _ = val
1217
+
1218
+ result = s(:defined, arg).line line
1101
1219
  }
1102
1220
  | kNOT tLPAREN2 expr rparen
1103
1221
  {
1104
- result = s(:call, val[2], :"!")
1222
+ _, _, lhs, _ = val
1223
+ result = new_call lhs, :"!"
1105
1224
  }
1106
1225
  | kNOT tLPAREN2 rparen
1107
1226
  {
@@ -1109,11 +1228,11 @@ rule
1109
1228
  }
1110
1229
  | fcall brace_block
1111
1230
  {
1112
- oper, iter = val[0], val[1]
1113
- call = oper # FIX
1231
+ call, iter = val
1232
+
1114
1233
  iter.insert 1, call
1115
1234
  result = iter
1116
- call.line = iter.line
1235
+ # FIX: probably not: call.line = iter.line
1117
1236
  }
1118
1237
  | method_call
1119
1238
  | method_call brace_block
@@ -1221,66 +1340,82 @@ rule
1221
1340
  }
1222
1341
  | k_def fname
1223
1342
  {
1224
- result = [self.in_def, self.lexer.cmdarg.stack.dup]
1343
+ result = self.in_def
1225
1344
 
1226
- self.comments.push self.lexer.comments
1227
- self.in_def = true
1345
+ self.in_def = true # group = local_push
1228
1346
  self.env.extend
1229
- # TODO: local->cmdargs = cmdarg_stack;
1230
- # TODO: port local_push_gen and local_pop_gen
1231
- lexer.cmdarg.stack.replace [false]
1347
+ lexer.cmdarg.push false
1348
+ lexer.cond.push false
1349
+
1350
+ self.comments.push self.lexer.comments
1232
1351
  }
1233
- f_arglist bodystmt k_end
1352
+ f_arglist bodystmt { result = lexer.lineno } k_end
1234
1353
  {
1235
- in_def, cmdarg = val[2]
1354
+ in_def = val[2]
1236
1355
 
1237
1356
  result = new_defn val
1238
1357
 
1239
- lexer.cmdarg.stack.replace cmdarg
1358
+ lexer.cond.pop # group = local_pop
1359
+ lexer.cmdarg.pop
1240
1360
  self.env.unextend
1241
1361
  self.in_def = in_def
1362
+
1242
1363
  self.lexer.comments # we don't care about comments in the body
1243
1364
  }
1244
1365
  | k_def singleton dot_or_colon
1245
1366
  {
1246
- self.comments.push self.lexer.comments
1247
1367
  lexer.lex_state = EXPR_FNAME
1248
1368
  }
1249
1369
  fname
1250
1370
  {
1251
- self.in_single += 1
1371
+ result = [self.in_def, lexer.lineno]
1372
+
1373
+ self.in_single += 1 # TODO: remove?
1374
+
1375
+ self.in_def = true # local_push
1252
1376
  self.env.extend
1253
- lexer.lex_state = EXPR_ENDFN # force for args
1254
- result = [lexer.lineno, self.lexer.cmdarg.stack.dup]
1255
- lexer.cmdarg.stack.replace [false]
1377
+ lexer.cmdarg.push false
1378
+ lexer.cond.push false
1379
+
1380
+ lexer.lex_state = EXPR_ENDFN|EXPR_LABEL
1381
+ self.comments.push self.lexer.comments
1256
1382
  }
1257
1383
  f_arglist bodystmt k_end
1258
1384
  {
1259
- line, cmdarg = val[5]
1260
- result = new_defs val
1261
- result[3].line line
1385
+ _, _recv, _, _, _name, (in_def, _lineno), _args, _body, _ = val
1262
1386
 
1263
- lexer.cmdarg.stack.replace cmdarg
1387
+ result = new_defs val
1264
1388
 
1389
+ lexer.cond.pop # group = local_pop
1390
+ lexer.cmdarg.pop
1265
1391
  self.env.unextend
1392
+ self.in_def = in_def
1393
+
1266
1394
  self.in_single -= 1
1395
+
1396
+ # TODO: restore cur_arg ? what's cur_arg?
1397
+
1267
1398
  self.lexer.comments # we don't care about comments in the body
1268
1399
  }
1269
1400
  | kBREAK
1270
1401
  {
1271
- result = s(:break)
1402
+ (_, line), = val
1403
+ result = s(:break).line line
1272
1404
  }
1273
1405
  | kNEXT
1274
1406
  {
1275
- result = s(:next)
1407
+ (_, line), = val
1408
+ result = s(:next).line line
1276
1409
  }
1277
1410
  | kREDO
1278
1411
  {
1279
- result = s(:redo)
1412
+ (_, line), = val
1413
+ result = s(:redo).line line
1280
1414
  }
1281
1415
  | kRETRY
1282
1416
  {
1283
- result = s(:retry)
1417
+ (_, line), = val
1418
+ result = s(:retry).line line
1284
1419
  }
1285
1420
 
1286
1421
  primary_value: primary
@@ -1319,7 +1454,9 @@ rule
1319
1454
  if_tail: opt_else
1320
1455
  | k_elsif expr_value then compstmt if_tail
1321
1456
  {
1322
- result = s(:if, val[1], val[3], val[4])
1457
+ (_, line), c, _, t, rest = val
1458
+
1459
+ result = s(:if, c, t, rest).line line
1323
1460
  }
1324
1461
 
1325
1462
  opt_else: none
@@ -1342,7 +1479,9 @@ rule
1342
1479
 
1343
1480
  f_marg_list: f_marg
1344
1481
  {
1345
- result = s(:array, val[0])
1482
+ sym, = val
1483
+
1484
+ result = s(:array, sym).line lexer.lineno
1346
1485
  }
1347
1486
  | f_marg_list tCOMMA f_marg
1348
1487
  {
@@ -1416,7 +1555,9 @@ rule
1416
1555
  }
1417
1556
  | f_block_arg
1418
1557
  {
1419
- result = call_args val
1558
+ line = lexer.lineno
1559
+ result = call_args val # TODO: push line down
1560
+ result.line line
1420
1561
  }
1421
1562
 
1422
1563
  opt_block_args_tail: tCOMMA block_args_tail
@@ -1447,7 +1588,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1447
1588
  }
1448
1589
  | f_arg tCOMMA
1449
1590
  {
1450
- result = args val
1591
+ result = args(val) << nil
1451
1592
  }
1452
1593
  | f_arg tCOMMA f_rest_arg tCOMMA f_arg opt_block_args_tail
1453
1594
  {
@@ -1499,7 +1640,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1499
1640
  }
1500
1641
  | tOROP
1501
1642
  {
1502
- result = s(:args)
1643
+ result = s(:args).line lexer.lineno
1503
1644
  }
1504
1645
  | tPIPE block_param opt_bv_decl tPIPE
1505
1646
  {
@@ -1524,34 +1665,33 @@ opt_block_args_tail: tCOMMA block_args_tail
1524
1665
 
1525
1666
  bvar: tIDENTIFIER
1526
1667
  {
1527
- result = s(:shadow, val[0].to_sym)
1668
+ id, = val
1669
+ line = lexer.lineno
1670
+ result = s(:shadow, id.to_sym).line line
1528
1671
  }
1529
1672
  | f_bad_arg
1530
1673
 
1531
1674
  lambda: {
1532
1675
  self.env.extend :dynamic
1533
- result = self.lexer.lineno
1534
-
1535
- result = lexer.lpar_beg
1676
+ result = [lexer.lineno, lexer.lpar_beg]
1536
1677
  lexer.paren_nest += 1
1537
1678
  lexer.lpar_beg = lexer.paren_nest
1538
1679
  }
1539
1680
  f_larglist
1540
1681
  {
1541
- result = [lexer.cmdarg.store(false), self.lexer.lineno]
1682
+ lexer.cmdarg.push false
1542
1683
  }
1543
1684
  lambda_body
1544
1685
  {
1545
- lpar, args, (cmdarg, lineno), body = val
1686
+ (line, lpar), args, _cmdarg, body = val
1546
1687
  lexer.lpar_beg = lpar
1547
1688
 
1548
- lexer.cmdarg.restore cmdarg
1549
- lexer.cmdarg.lexpop
1689
+ lexer.cmdarg.pop
1550
1690
 
1551
- call = new_call nil, :lambda
1691
+ call = s(:lambda).line line
1552
1692
  result = new_iter call, args, body
1553
- result.line = lineno
1554
- self.env.unextend
1693
+ result.line = line
1694
+ self.env.unextend # TODO: dynapush & dynapop
1555
1695
  }
1556
1696
 
1557
1697
  f_larglist: tLPAREN2 f_args opt_bv_decl rparen
@@ -1575,8 +1715,8 @@ opt_block_args_tail: tCOMMA block_args_tail
1575
1715
 
1576
1716
  do_block: k_do_block do_body kEND
1577
1717
  {
1578
- # TODO: maybe fix lineno to kDO's lineno?
1579
- result = val[1]
1718
+ (_, line), iter, _ = val
1719
+ result = iter.line line
1580
1720
  }
1581
1721
 
1582
1722
  block_call: command do_block
@@ -1590,8 +1730,10 @@ opt_block_args_tail: tCOMMA block_args_tail
1590
1730
 
1591
1731
  val = invert_block_call val if inverted? val
1592
1732
 
1593
- result = val[1]
1594
- result.insert 1, val[0]
1733
+ cmd, blk = val
1734
+
1735
+ result = blk
1736
+ result.insert 1, cmd
1595
1737
  }
1596
1738
  | block_call call_op2 operation2 opt_paren_args
1597
1739
  {
@@ -1622,8 +1764,10 @@ opt_block_args_tail: tCOMMA block_args_tail
1622
1764
  }
1623
1765
  paren_args
1624
1766
  {
1625
- args = self.call_args val[2..-1]
1626
- result = val[0].concat args.sexp_body
1767
+ call, lineno, args = val
1768
+
1769
+ result = call.concat args.sexp_body if args
1770
+ result.line lineno
1627
1771
  }
1628
1772
  | primary_value call_op operation2 opt_paren_args
1629
1773
  {
@@ -1651,7 +1795,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1651
1795
  }
1652
1796
  | kSUPER
1653
1797
  {
1654
- result = s(:zsuper)
1798
+ result = s(:zsuper).line lexer.lineno
1655
1799
  }
1656
1800
  | primary_value tLBRACK2 opt_call_args rbracket
1657
1801
  {
@@ -1700,15 +1844,15 @@ opt_block_args_tail: tCOMMA block_args_tail
1700
1844
  }
1701
1845
 
1702
1846
  do_body: { self.env.extend :dynamic; result = self.lexer.lineno }
1703
- { result = lexer.cmdarg.store(false) }
1847
+ { lexer.cmdarg.push false }
1704
1848
  opt_block_param
1705
1849
  compstmt
1706
1850
  {
1707
- line, cmdarg, param, cmpstmt = val
1851
+ line, _cmdarg, param, cmpstmt = val
1708
1852
 
1709
1853
  result = new_do_body param, cmpstmt, line
1854
+ lexer.cmdarg.pop
1710
1855
  self.env.unextend
1711
- lexer.cmdarg.restore cmdarg
1712
1856
  }
1713
1857
 
1714
1858
  case_body: k_when
@@ -1729,7 +1873,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1729
1873
  (_, line), klasses, var, _, body, rest = val
1730
1874
 
1731
1875
  klasses ||= s(:array)
1732
- klasses << new_assign(var, s(:gvar, :"$!")) if var
1876
+ klasses << new_assign(var, s(:gvar, :"$!").line(var.line)) if var
1733
1877
  klasses.line line
1734
1878
 
1735
1879
  result = new_resbody(klasses, body)
@@ -1742,7 +1886,8 @@ opt_block_args_tail: tCOMMA block_args_tail
1742
1886
 
1743
1887
  exc_list: arg_value
1744
1888
  {
1745
- result = s(:array, val[0])
1889
+ arg, = val
1890
+ result = s(:array, arg).line arg.line
1746
1891
  }
1747
1892
  | mrhs
1748
1893
  | none
@@ -1755,26 +1900,31 @@ opt_block_args_tail: tCOMMA block_args_tail
1755
1900
 
1756
1901
  opt_ensure: k_ensure compstmt
1757
1902
  {
1758
- _, body = val
1903
+ (_, line), body = val
1759
1904
 
1760
- result = body || s(:nil)
1905
+ result = body || s(:nil).line(line)
1761
1906
  }
1762
1907
  | none
1763
1908
 
1764
1909
  literal: numeric
1765
1910
  {
1911
+ line = lexer.lineno
1766
1912
  result = s(:lit, val[0])
1913
+ result.line = line
1767
1914
  }
1768
1915
  | symbol
1769
1916
  {
1917
+ line = lexer.lineno
1770
1918
  result = s(:lit, val[0])
1919
+ result.line = line
1771
1920
  }
1772
1921
  | dsym
1773
1922
 
1774
1923
  strings: string
1775
1924
  {
1776
- val[0] = s(:dstr, val[0].value) if val[0].sexp_type == :evstr
1777
- result = val[0]
1925
+ str, = val
1926
+ str = s(:dstr, str.value) if str.sexp_type == :evstr
1927
+ result = str
1778
1928
  }
1779
1929
 
1780
1930
  string: tCHAR
@@ -1789,7 +1939,11 @@ opt_block_args_tail: tCOMMA block_args_tail
1789
1939
 
1790
1940
  string1: tSTRING_BEG string_contents tSTRING_END
1791
1941
  {
1792
- result = val[1]
1942
+ _, str, (_, func) = val
1943
+
1944
+ str = dedent str if func =~ RubyLexer::STR_FUNC_ICNTNT
1945
+
1946
+ result = str
1793
1947
  }
1794
1948
  | tSTRING
1795
1949
  {
@@ -1798,7 +1952,8 @@ opt_block_args_tail: tCOMMA block_args_tail
1798
1952
 
1799
1953
  xstring: tXSTRING_BEG xstring_contents tSTRING_END
1800
1954
  {
1801
- result = new_xstring val[1]
1955
+ result = new_xstring val
1956
+ # TODO: dedent?!?! SERIOUSLY?!?
1802
1957
  }
1803
1958
 
1804
1959
  regexp: tREGEXP_BEG regexp_contents tREGEXP_END
@@ -1808,7 +1963,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1808
1963
 
1809
1964
  words: tWORDS_BEG tSPACE tSTRING_END
1810
1965
  {
1811
- result = s(:array)
1966
+ result = s(:array).line lexer.lineno
1812
1967
  }
1813
1968
  | tWORDS_BEG word_list tSTRING_END
1814
1969
  {
@@ -1832,25 +1987,28 @@ opt_block_args_tail: tCOMMA block_args_tail
1832
1987
 
1833
1988
  symbols: tSYMBOLS_BEG tSPACE tSTRING_END
1834
1989
  {
1835
- result = s(:array)
1990
+ result = s(:array).line lexer.lineno
1836
1991
  }
1837
- | tSYMBOLS_BEG symbol_list tSTRING_END
1992
+ | tSYMBOLS_BEG { result = lexer.lineno } symbol_list tSTRING_END
1838
1993
  {
1839
- result = val[1]
1994
+ _, line, list, _, = val
1995
+ list.line = line
1996
+ result = list
1840
1997
  }
1841
1998
 
1842
1999
  symbol_list: none
1843
2000
  {
1844
- result = new_symbol_list
2001
+ result = new_symbol_list.line lexer.lineno
1845
2002
  }
1846
2003
  | symbol_list word tSPACE
1847
2004
  {
1848
- result = val[0].dup << new_symbol_list_entry(val)
2005
+ list, * = val
2006
+ result = list.dup << new_symbol_list_entry(val)
1849
2007
  }
1850
2008
 
1851
2009
  qwords: tQWORDS_BEG tSPACE tSTRING_END
1852
2010
  {
1853
- result = s(:array)
2011
+ result = s(:array).line lexer.lineno
1854
2012
  }
1855
2013
  | tQWORDS_BEG qword_list tSTRING_END
1856
2014
  {
@@ -1859,7 +2017,7 @@ opt_block_args_tail: tCOMMA block_args_tail
1859
2017
 
1860
2018
  qsymbols: tQSYMBOLS_BEG tSPACE tSTRING_END
1861
2019
  {
1862
- result = s(:array)
2020
+ result = s(:array).line lexer.lineno # FIX
1863
2021
  }
1864
2022
  | tQSYMBOLS_BEG qsym_list tSTRING_END
1865
2023
  {
@@ -1886,11 +2044,12 @@ opt_block_args_tail: tCOMMA block_args_tail
1886
2044
 
1887
2045
  string_contents: none
1888
2046
  {
1889
- result = s(:str, "")
2047
+ result = s(:str, "").line lexer.lineno
1890
2048
  }
1891
2049
  | string_contents string_content
1892
2050
  {
1893
- result = literal_concat(val[0], val[1])
2051
+ v1, v2 = val
2052
+ result = literal_concat v1, v2
1894
2053
  }
1895
2054
 
1896
2055
  xstring_contents: none
@@ -1899,7 +2058,8 @@ xstring_contents: none
1899
2058
  }
1900
2059
  | xstring_contents string_content
1901
2060
  {
1902
- result = literal_concat(val[0], val[1])
2061
+ v1, v2 = val
2062
+ result = literal_concat v1, v2
1903
2063
  }
1904
2064
 
1905
2065
  regexp_contents: none
@@ -1908,7 +2068,8 @@ regexp_contents: none
1908
2068
  }
1909
2069
  | regexp_contents string_content
1910
2070
  {
1911
- result = literal_concat(val[0], val[1])
2071
+ v1, v2 = val
2072
+ result = literal_concat v1, v2
1912
2073
  }
1913
2074
 
1914
2075
  string_content: tSTRING_CONTENT
@@ -1924,19 +2085,22 @@ regexp_contents: none
1924
2085
  }
1925
2086
  string_dvar
1926
2087
  {
1927
- lexer.lex_strterm = val[1]
1928
- result = s(:evstr, val[2])
2088
+ _, strterm, str = val
2089
+ lexer.lex_strterm = strterm
2090
+ result = s(:evstr, str).line str.line
1929
2091
  }
1930
2092
  | tSTRING_DBEG
1931
2093
  {
1932
2094
  result = [lexer.lex_strterm,
1933
2095
  lexer.brace_nest,
1934
2096
  lexer.string_nest, # TODO: remove
1935
- lexer.cond.store,
1936
- lexer.cmdarg.store,
1937
2097
  lexer.lex_state,
2098
+ lexer.lineno,
1938
2099
  ]
1939
2100
 
2101
+ lexer.cmdarg.push false
2102
+ lexer.cond.push false
2103
+
1940
2104
  lexer.lex_strterm = nil
1941
2105
  lexer.brace_nest = 0
1942
2106
  lexer.string_nest = 0
@@ -1948,14 +2112,15 @@ regexp_contents: none
1948
2112
  {
1949
2113
  _, memo, stmt, _ = val
1950
2114
 
1951
- lex_strterm, brace_nest, string_nest, oldcond, oldcmdarg, oldlex_state = memo
2115
+ lex_strterm, brace_nest, string_nest, oldlex_state, line = memo
2116
+ # TODO: heredoc_indent
1952
2117
 
1953
2118
  lexer.lex_strterm = lex_strterm
1954
2119
  lexer.brace_nest = brace_nest
1955
2120
  lexer.string_nest = string_nest
1956
2121
 
1957
- lexer.cond.restore oldcond
1958
- lexer.cmdarg.restore oldcmdarg
2122
+ lexer.cmdarg.pop
2123
+ lexer.cond.pop
1959
2124
 
1960
2125
  lexer.lex_state = oldlex_state
1961
2126
 
@@ -1965,19 +2130,19 @@ regexp_contents: none
1965
2130
  when :str, :dstr, :evstr then
1966
2131
  result = stmt
1967
2132
  else
1968
- result = s(:evstr, stmt)
2133
+ result = s(:evstr, stmt).line line
1969
2134
  end
1970
2135
  when nil then
1971
- result = s(:evstr)
2136
+ result = s(:evstr).line line
1972
2137
  else
1973
2138
  debug20 25
1974
2139
  raise "unknown string body: #{stmt.inspect}"
1975
2140
  end
1976
2141
  }
1977
2142
 
1978
- string_dvar: tGVAR { result = s(:gvar, val[0].to_sym) }
1979
- | tIVAR { result = s(:ivar, val[0].to_sym) }
1980
- | tCVAR { result = s(:cvar, val[0].to_sym) }
2143
+ string_dvar: tGVAR { result = s(:gvar, val[0].to_sym).line lexer.lineno }
2144
+ | tIVAR { result = s(:ivar, val[0].to_sym).line lexer.lineno }
2145
+ | tCVAR { result = s(:cvar, val[0].to_sym).line lexer.lineno }
1981
2146
  | backref
1982
2147
 
1983
2148
  symbol: tSYMBEG sym
@@ -1994,18 +2159,19 @@ regexp_contents: none
1994
2159
 
1995
2160
  dsym: tSYMBEG xstring_contents tSTRING_END
1996
2161
  {
2162
+ _, result, _ = val
2163
+
1997
2164
  lexer.lex_state = EXPR_END
1998
- result = val[1]
1999
2165
 
2000
- result ||= s(:str, "")
2166
+ result ||= s(:str, "").line lexer.lineno
2001
2167
 
2002
2168
  case result.sexp_type
2003
2169
  when :dstr then
2004
2170
  result.sexp_type = :dsym
2005
2171
  when :str then
2006
- result = s(:lit, result.last.to_sym)
2172
+ result = s(:lit, result.last.to_sym).line result.line
2007
2173
  when :evstr then
2008
- result = s(:dsym, "", result)
2174
+ result = s(:dsym, "", result).line result.line
2009
2175
  else
2010
2176
  debug20 26, val, result
2011
2177
  end
@@ -2028,19 +2194,20 @@ regexp_contents: none
2028
2194
  | tCONSTANT
2029
2195
  | tCVAR
2030
2196
 
2031
- keyword_variable: kNIL { result = s(:nil) }
2032
- | kSELF { result = s(:self) }
2033
- | kTRUE { result = s(:true) }
2034
- | kFALSE { result = s(:false) }
2035
- | k__FILE__ { result = s(:str, self.file) }
2036
- | k__LINE__ { result = s(:lit, lexer.lineno) }
2197
+ keyword_variable: kNIL { result = s(:nil).line lexer.lineno }
2198
+ | kSELF { result = s(:self).line lexer.lineno }
2199
+ | kTRUE { result = s(:true).line lexer.lineno }
2200
+ | kFALSE { result = s(:false).line lexer.lineno }
2201
+ | k__FILE__ { result = s(:str, self.file).line lexer.lineno }
2202
+ | k__LINE__ { result = s(:lit, lexer.lineno).line lexer.lineno }
2037
2203
  | k__ENCODING__
2038
2204
  {
2205
+ l = lexer.lineno
2039
2206
  result =
2040
2207
  if defined? Encoding then
2041
- s(:colon2, s(:const, :Encoding), :UTF_8)
2208
+ s(:colon2, s(:const, :Encoding).line(l), :UTF_8).line l
2042
2209
  else
2043
- s(:str, "Unsupported!")
2210
+ s(:str, "Unsupported!").line l
2044
2211
  end
2045
2212
  }
2046
2213
 
@@ -2065,8 +2232,8 @@ keyword_variable: kNIL { result = s(:nil) }
2065
2232
  debug20 29, val, result
2066
2233
  }
2067
2234
 
2068
- backref: tNTH_REF { result = s(:nth_ref, val[0]) }
2069
- | tBACK_REF { result = s(:back_ref, val[0]) }
2235
+ backref: tNTH_REF { result = s(:nth_ref, val[0]).line lexer.lineno }
2236
+ | tBACK_REF { result = s(:back_ref, val[0]).line lexer.lineno }
2070
2237
 
2071
2238
  superclass: tLT
2072
2239
  {
@@ -2223,12 +2390,13 @@ keyword_variable: kNIL { result = s(:nil) }
2223
2390
 
2224
2391
  f_arg: f_arg_item
2225
2392
  {
2226
- case val[0]
2393
+ arg, = val
2394
+
2395
+ case arg
2227
2396
  when Symbol then
2228
- result = s(:args)
2229
- result << val[0]
2397
+ result = s(:args, arg).line lexer.lineno
2230
2398
  when Sexp then
2231
- result = val[0]
2399
+ result = arg
2232
2400
  else
2233
2401
  debug20 32
2234
2402
  raise "Unknown f_arg type: #{val.inspect}"
@@ -2241,7 +2409,7 @@ keyword_variable: kNIL { result = s(:nil) }
2241
2409
  if list.sexp_type == :args then
2242
2410
  result = list
2243
2411
  else
2244
- result = s(:args, list)
2412
+ result = s(:args, list).line list.line
2245
2413
  end
2246
2414
 
2247
2415
  result << item
@@ -2251,38 +2419,42 @@ keyword_variable: kNIL { result = s(:nil) }
2251
2419
 
2252
2420
  f_kw: f_label arg_value
2253
2421
  {
2254
- # TODO: call_args
2255
- label, _ = val[0] # TODO: fix lineno?
2422
+ # TODO: new_kw_arg
2423
+ (label, line), arg = val
2424
+
2256
2425
  identifier = label.to_sym
2257
2426
  self.env[identifier] = :lvar
2258
2427
 
2259
- result = s(:array, s(:kwarg, identifier, val[1]))
2428
+ kwarg = s(:kwarg, identifier, arg).line line
2429
+ result = s(:array, kwarg).line line
2260
2430
  }
2261
2431
  | f_label
2262
2432
  {
2263
- label, _ = val[0] # TODO: fix lineno?
2264
- identifier = label.to_sym
2265
- self.env[identifier] = :lvar
2433
+ (label, line), = val
2266
2434
 
2267
- result = s(:array, s(:kwarg, identifier))
2435
+ id = label.to_sym
2436
+ self.env[id] = :lvar
2437
+
2438
+ result = s(:array, s(:kwarg, id).line(line)).line line
2268
2439
  }
2269
2440
 
2270
2441
  f_block_kw: f_label primary_value
2271
2442
  {
2272
- # TODO: call_args
2273
- label, _ = val[0] # TODO: fix lineno?
2274
- identifier = label.to_sym
2275
- self.env[identifier] = :lvar
2443
+ # TODO: new_kw_arg
2444
+ (label, line), expr = val
2445
+ id = label.to_sym
2446
+ self.env[id] = :lvar
2276
2447
 
2277
- result = s(:array, s(:kwarg, identifier, val[1]))
2448
+ result = s(:array, s(:kwarg, id, expr).line(line)).line line
2278
2449
  }
2279
2450
  | f_label
2280
2451
  {
2281
- label, _ = val[0] # TODO: fix lineno?
2282
- identifier = label.to_sym
2283
- self.env[identifier] = :lvar
2452
+ # TODO: new_kw_arg
2453
+ (label, line), = val
2454
+ id = label.to_sym
2455
+ self.env[id] = :lvar
2284
2456
 
2285
- result = s(:array, s(:kwarg, identifier))
2457
+ result = s(:array, s(:kwarg, id).line(line)).line line
2286
2458
  }
2287
2459
 
2288
2460
  f_block_kwarg: f_block_kw
@@ -2325,17 +2497,20 @@ keyword_variable: kNIL { result = s(:nil) }
2325
2497
 
2326
2498
  f_block_optarg: f_block_opt
2327
2499
  {
2328
- result = s(:block, val[0])
2500
+ optblk, = val
2501
+ result = s(:block, optblk).line optblk.line
2329
2502
  }
2330
2503
  | f_block_optarg tCOMMA f_block_opt
2331
2504
  {
2332
- result = val[0]
2333
- result << val[2]
2505
+ optarg, _, optblk = val
2506
+ result = optarg
2507
+ result << optblk
2334
2508
  }
2335
2509
 
2336
2510
  f_optarg: f_opt
2337
2511
  {
2338
- result = s(:block, val[0])
2512
+ opt, = val
2513
+ result = s(:block, opt).line opt.line
2339
2514
  }
2340
2515
  | f_optarg tCOMMA f_opt
2341
2516
  {
@@ -2389,14 +2564,11 @@ keyword_variable: kNIL { result = s(:nil) }
2389
2564
  result.sexp_type == :lit
2390
2565
  }
2391
2566
 
2392
- assoc_list: none # [!nil]
2567
+ assoc_list: none
2393
2568
  {
2394
- result = s(:array)
2395
- }
2396
- | assocs trailer # [!nil]
2397
- {
2398
- result = val[0]
2569
+ result = s(:array).line lexer.lineno
2399
2570
  }
2571
+ | assocs trailer
2400
2572
 
2401
2573
  assocs: assoc
2402
2574
  | assocs tCOMMA assoc
@@ -2410,22 +2582,27 @@ keyword_variable: kNIL { result = s(:nil) }
2410
2582
 
2411
2583
  assoc: arg_value tASSOC arg_value
2412
2584
  {
2413
- result = s(:array, val[0], val[2])
2585
+ v1, _, v2 = val
2586
+ result = s(:array, v1, v2).line v1.line
2414
2587
  }
2415
2588
  | tLABEL arg_value
2416
2589
  {
2417
- (label, _), arg = val
2418
- result = s(:array, s(:lit, label.to_sym), arg)
2590
+ (label, line), arg = val
2591
+
2592
+ lit = s(:lit, label.to_sym).line line
2593
+ result = s(:array, lit, arg).line line
2419
2594
  }
2420
2595
  | tSTRING_BEG string_contents tLABEL_END arg_value
2421
2596
  {
2422
2597
  _, sym, _, value = val
2423
2598
  sym.sexp_type = :dsym
2424
- result = s(:array, sym, value)
2599
+ result = s(:array, sym, value).line sym.line
2425
2600
  }
2426
2601
  | tDSTAR arg_value
2427
2602
  {
2428
- result = s(:array, s(:kwsplat, val[1]))
2603
+ _, arg = val
2604
+ line = arg.line
2605
+ result = s(:array, s(:kwsplat, arg).line(line)).line line
2429
2606
  }
2430
2607
 
2431
2608
  operation: tIDENTIFIER | tCONSTANT | tFID