ruby_parser 3.15.1 → 3.18.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/History.rdoc +107 -0
  4. data/Manifest.txt +5 -0
  5. data/README.rdoc +1 -0
  6. data/Rakefile +137 -29
  7. data/bin/ruby_parse_extract_error +1 -1
  8. data/compare/normalize.rb +8 -3
  9. data/debugging.md +133 -0
  10. data/gauntlet.md +106 -0
  11. data/lib/rp_extensions.rb +15 -36
  12. data/lib/rp_stringscanner.rb +20 -51
  13. data/lib/ruby20_parser.rb +3568 -3502
  14. data/lib/ruby20_parser.y +342 -251
  15. data/lib/ruby21_parser.rb +3659 -3617
  16. data/lib/ruby21_parser.y +337 -248
  17. data/lib/ruby22_parser.rb +3699 -3631
  18. data/lib/ruby22_parser.y +341 -250
  19. data/lib/ruby23_parser.rb +3638 -3576
  20. data/lib/ruby23_parser.y +341 -250
  21. data/lib/ruby24_parser.rb +3721 -3657
  22. data/lib/ruby24_parser.y +341 -250
  23. data/lib/ruby25_parser.rb +3721 -3657
  24. data/lib/ruby25_parser.y +341 -250
  25. data/lib/ruby26_parser.rb +3724 -3661
  26. data/lib/ruby26_parser.y +341 -249
  27. data/lib/ruby27_parser.rb +5018 -3725
  28. data/lib/ruby27_parser.y +937 -248
  29. data/lib/ruby30_parser.rb +8751 -0
  30. data/lib/ruby30_parser.y +3472 -0
  31. data/lib/ruby3_parser.yy +3476 -0
  32. data/lib/ruby_lexer.rb +273 -602
  33. data/lib/ruby_lexer.rex +28 -21
  34. data/lib/ruby_lexer.rex.rb +60 -24
  35. data/lib/ruby_lexer_strings.rb +638 -0
  36. data/lib/ruby_parser.rb +2 -0
  37. data/lib/ruby_parser.yy +978 -255
  38. data/lib/ruby_parser_extras.rb +297 -116
  39. data/test/test_ruby_lexer.rb +213 -129
  40. data/test/test_ruby_parser.rb +1479 -281
  41. data/tools/munge.rb +36 -8
  42. data/tools/ripper.rb +15 -10
  43. data.tar.gz.sig +0 -0
  44. metadata +36 -23
  45. metadata.gz.sig +0 -0
data/test/test_ruby_lexer.rb CHANGED
@@ -46,11 +46,21 @@ class TestRubyLexer < Minitest::Test
46
46
  end
47
47
 
48
48
  def assert_lex3 input, exp_sexp, *args, &block
49
+ # TODO: refute_nil exp_sexp, "Get off your lazy butt and write one"
50
+
49
51
  args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
50
52
 
51
53
  assert_lex(input, exp_sexp, *args, &block)
52
54
  end
53
55
 
56
+ def refute_lex3 input, *args # TODO: re-sort
57
+ args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
58
+
59
+ assert_raises RubyParser::SyntaxError do
60
+ assert_lex(input, nil, *args)
61
+ end
62
+ end
63
+
54
64
  def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
55
65
  assert_lex3("def #{name} ",
56
66
  nil,
@@ -95,15 +105,15 @@ class TestRubyLexer < Minitest::Test
95
105
  end
96
106
 
97
107
  def assert_read_escape expected, input
98
- @lex.ss.string = input.dup
108
+ setup_lexer input
99
109
  enc = expected.encoding
100
- assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
110
+ assert_equal expected, lex.read_escape.b.force_encoding(enc), input
101
111
  end
102
112
 
103
113
  def assert_read_escape_bad input # TODO: rename refute_read_escape
104
- @lex.ss.string = input
114
+ setup_lexer input
105
115
  assert_raises RubyParser::SyntaxError do
106
- @lex.read_escape
116
+ lex.read_escape
107
117
  end
108
118
  end
109
119
 
@@ -124,7 +134,7 @@ class TestRubyLexer < Minitest::Test
124
134
  def refute_lexeme
125
135
  x = y = @lex.next_token
126
136
 
127
- refute x, "not empty: #{y.inspect}"
137
+ refute x, "not empty: #{y.inspect}: #{@lex.rest.inspect}"
128
138
  end
129
139
 
130
140
  ## Utility Methods:
@@ -159,8 +169,8 @@ class TestRubyLexer < Minitest::Test
159
169
  ## Tests:
160
170
 
161
171
  def test_next_token
162
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
163
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
172
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
173
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
164
174
  assert_nil @lex.next_token
165
175
  end
166
176
 
@@ -173,10 +183,10 @@ class TestRubyLexer < Minitest::Test
173
183
 
174
184
  :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
175
185
  :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
176
- :tSPACE, nil, EXPR_BEG, 0, 0,
186
+ :tSPACE, " ", EXPR_BEG, 0, 0,
177
187
  :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
178
- :tSPACE, nil, EXPR_BEG, 0, 0,
179
- :tSTRING_END, nil, EXPR_LIT, 0, 0)
188
+ :tSPACE, "]", EXPR_BEG, 0, 0,
189
+ :tSTRING_END, "]", EXPR_LIT, 0, 0)
180
190
  end
181
191
  end
182
192
 
@@ -478,6 +488,22 @@ class TestRubyLexer < Minitest::Test
478
488
  assert_lex3("!~", nil, :tNMATCH, "!~", EXPR_BEG)
479
489
  end
480
490
 
491
+ def test_yylex_bdot2
492
+ assert_lex3("..42",
493
+ nil, # TODO: s(:dot2, nil, s(:lit, 42)),
494
+
495
+ :tBDOT2, "..", EXPR_BEG,
496
+ :tINTEGER, 42, EXPR_END|EXPR_ENDARG)
497
+ end
498
+
499
+ def test_yylex_bdot3
500
+ assert_lex3("...42",
501
+ nil, # TODO: s(:dot2, nil, s(:lit, 42)),
502
+
503
+ :tBDOT3, "...", EXPR_BEG,
504
+ :tINTEGER, 42, EXPR_END|EXPR_ENDARG)
505
+ end
506
+
481
507
  def test_yylex_block_bug_1
482
508
  assert_lex3("a do end",
483
509
  s(:iter, s(:call, nil, :a), 0),
@@ -670,8 +696,8 @@ class TestRubyLexer < Minitest::Test
670
696
  end
671
697
 
672
698
  def test_yylex_def_bad_name
673
- self.lex_state = EXPR_FNAME
674
- refute_lex("def [ ", :kDEF, "def")
699
+ refute_lex3("def [ ",
700
+ :kDEF, "def", EXPR_FNAME)
675
701
  end
676
702
 
677
703
  def test_yylex_div
@@ -740,19 +766,31 @@ class TestRubyLexer < Minitest::Test
740
766
  assert_includes(e.message, "is not allowed as a global variable name")
741
767
  end
742
768
 
743
- def test_yylex_dollar_eos
744
- assert_lex3("$", nil, "$", "$", EXPR_END) # FIX: wtf is this?!?
745
- end
746
-
747
769
  def test_yylex_dot # HINT message sends
748
770
  assert_lex3(".", nil, :tDOT, ".", EXPR_DOT)
749
771
  end
750
772
 
751
773
  def test_yylex_dot2
774
+ assert_lex3("1..2",
775
+ s(:lit, 1..2),
776
+
777
+ :tINTEGER, 1, EXPR_END|EXPR_ENDARG,
778
+ :tDOT2, "..", EXPR_BEG,
779
+ :tINTEGER, 2, EXPR_END|EXPR_ENDARG)
780
+
781
+ self.lex_state = EXPR_END|EXPR_ENDARG
752
782
  assert_lex3("..", nil, :tDOT2, "..", EXPR_BEG)
753
783
  end
754
784
 
755
785
  def test_yylex_dot3
786
+ assert_lex3("1...2",
787
+ s(:lit, 1...2),
788
+
789
+ :tINTEGER, 1, EXPR_END|EXPR_ENDARG,
790
+ :tDOT3, "...", EXPR_BEG,
791
+ :tINTEGER, 2, EXPR_END|EXPR_ENDARG)
792
+
793
+ self.lex_state = EXPR_END|EXPR_ENDARG
756
794
  assert_lex3("...", nil, :tDOT3, "...", EXPR_BEG)
757
795
  end
758
796
 
@@ -906,6 +944,9 @@ class TestRubyLexer < Minitest::Test
906
944
  end
907
945
 
908
946
  def test_yylex_global_dash_nothing
947
+ refute_lex3("$- ", nil) # fails 2.1+
948
+
949
+ setup_lexer_class RubyParser::V20
909
950
  assert_lex3("$- ", nil, :tGVAR, "$-", EXPR_END)
910
951
  end
911
952
 
@@ -1063,7 +1104,7 @@ class TestRubyLexer < Minitest::Test
1063
1104
  end
1064
1105
 
1065
1106
  def test_yylex_heredoc_backtick
1066
- assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1107
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n",
1067
1108
  nil,
1068
1109
  :tIDENTIFIER, "a", EXPR_CMDARG,
1069
1110
  :tEQL, "=", EXPR_BEG,
@@ -1085,21 +1126,41 @@ class TestRubyLexer < Minitest::Test
1085
1126
  end
1086
1127
 
1087
1128
  def test_yylex_heredoc_double_dash
1088
- assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1129
+ assert_lex3("a = \" blah blah\n\".strip\n42",
1130
+ nil,
1131
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1132
+ :tEQL, "=", EXPR_BEG,
1133
+ :tSTRING, " blah blah\n", EXPR_END,
1134
+ :tDOT, ".", EXPR_DOT,
1135
+ :tIDENTIFIER, "strip", EXPR_ARG,
1136
+ :tNL, nil, EXPR_BEG,
1137
+
1138
+ :tINTEGER, 42, EXPR_END
1139
+ )
1140
+
1141
+ assert_lex3("a = <<-\"EOF\".strip\n blah blah\n EOF\n42",
1089
1142
  nil,
1090
1143
  :tIDENTIFIER, "a", EXPR_CMDARG,
1091
1144
  :tEQL, "=", EXPR_BEG,
1092
1145
  :tSTRING_BEG, "\"", EXPR_BEG,
1093
1146
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1094
1147
  :tSTRING_END, "EOF", EXPR_LIT,
1095
- :tNL, nil, EXPR_BEG)
1148
+
1149
+ :tDOT, ".", EXPR_DOT,
1150
+ :tIDENTIFIER, "strip", EXPR_ARG,
1151
+
1152
+ :tNL, nil, EXPR_BEG,
1153
+
1154
+ :tINTEGER, 42, EXPR_END
1155
+ )
1096
1156
  end
1097
1157
 
1098
1158
  def test_yylex_heredoc_double_eos
1099
1159
  refute_lex("a = <<\"EOF\"\nblah",
1100
1160
  :tIDENTIFIER, "a",
1101
1161
  :tEQL, "=",
1102
- :tSTRING_BEG, "\"")
1162
+ :tSTRING_BEG, "\"",
1163
+ :tSTRING_CONTENT, "blah")
1103
1164
  end
1104
1165
 
1105
1166
  def test_yylex_heredoc_double_eos_nl
@@ -1133,12 +1194,12 @@ class TestRubyLexer < Minitest::Test
1133
1194
 
1134
1195
  assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1135
1196
  nil,
1136
- :tIDENTIFIER, "a", EXPR_CMDARG,
1137
- :tEQL, "=", EXPR_BEG,
1138
- :tSTRING_BEG, "\"", EXPR_BEG,
1197
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1198
+ :tEQL, "=", EXPR_BEG,
1199
+ :tSTRING_BEG, "\"", EXPR_BEG,
1139
1200
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1140
- :tSTRING_END, "EOF", EXPR_LIT,
1141
- :tNL, nil, EXPR_BEG)
1201
+ :tSTRING_END, "EOF", EXPR_LIT,
1202
+ :tNL, nil, EXPR_BEG)
1142
1203
  end
1143
1204
 
1144
1205
  def test_yylex_heredoc_empty
@@ -1185,37 +1246,41 @@ class TestRubyLexer < Minitest::Test
1185
1246
 
1186
1247
  assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
1187
1248
  nil,
1188
- :tIDENTIFIER, "a", EXPR_CMDARG,
1189
- :tEQL, "=", EXPR_BEG,
1190
- :tSTRING_BEG, "\"", EXPR_BEG,
1249
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1250
+ :tEQL, "=", EXPR_BEG,
1251
+ :tSTRING_BEG, "\"", EXPR_BEG,
1191
1252
  :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1192
- :tSTRING_END, "EOF", EXPR_LIT,
1193
- :tNL, nil, EXPR_BEG)
1253
+ :tSTRING_END, "EOF", EXPR_LIT,
1254
+ :tNL, nil, EXPR_BEG)
1194
1255
  end
1195
1256
 
1196
1257
  def test_yylex_heredoc_single
1197
- assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1258
+ assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n\n\n42\n",
1198
1259
  nil,
1199
1260
  :tIDENTIFIER, "a", EXPR_CMDARG,
1200
1261
  :tEQL, "=", EXPR_BEG,
1201
- :tSTRING_BEG, "\"", EXPR_BEG,
1262
+ :tSTRING_BEG, "'", EXPR_BEG,
1202
1263
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1203
1264
  :tSTRING_END, "EOF", EXPR_LIT,
1265
+ :tNL, nil, EXPR_BEG,
1266
+ :tINTEGER, 42, EXPR_LIT,
1204
1267
  :tNL, nil, EXPR_BEG)
1268
+
1269
+ assert_nil lex.old_ss
1205
1270
  end
1206
1271
 
1207
1272
  def test_yylex_heredoc_single_bad_eos_body
1208
1273
  refute_lex("a = <<'EOF'\nblah",
1209
1274
  :tIDENTIFIER, "a",
1210
1275
  :tEQL, "=",
1211
- :tSTRING_BEG, "\"")
1276
+ :tSTRING_BEG, "'")
1212
1277
  end
1213
1278
 
1214
1279
  def test_yylex_heredoc_single_bad_eos_empty
1215
1280
  refute_lex("a = <<''\n",
1216
1281
  :tIDENTIFIER, "a",
1217
1282
  :tEQL, "=",
1218
- :tSTRING_BEG, "\"")
1283
+ :tSTRING_BEG, "'")
1219
1284
  end
1220
1285
 
1221
1286
  def test_yylex_heredoc_single_bad_eos_term
@@ -1237,7 +1302,7 @@ class TestRubyLexer < Minitest::Test
1237
1302
  nil,
1238
1303
  :tIDENTIFIER, "a", EXPR_CMDARG,
1239
1304
  :tEQL, "=", EXPR_BEG,
1240
- :tSTRING_BEG, "\"", EXPR_BEG,
1305
+ :tSTRING_BEG, "'", EXPR_BEG,
1241
1306
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1242
1307
  :tSTRING_END, "EOF", EXPR_LIT,
1243
1308
  :tNL, nil, EXPR_BEG)
@@ -1248,12 +1313,12 @@ class TestRubyLexer < Minitest::Test
1248
1313
 
1249
1314
  assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
1250
1315
  nil,
1251
- :tIDENTIFIER, "a", EXPR_CMDARG,
1252
- :tEQL, "=", EXPR_BEG,
1253
- :tSTRING_BEG, "\"", EXPR_BEG,
1316
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1317
+ :tEQL, "=", EXPR_BEG,
1318
+ :tSTRING_BEG, "'", EXPR_BEG,
1254
1319
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1255
- :tSTRING_END, "EOF", EXPR_LIT,
1256
- :tNL, nil, EXPR_BEG)
1320
+ :tSTRING_END, "EOF", EXPR_LIT,
1321
+ :tNL, nil, EXPR_BEG)
1257
1322
  end
1258
1323
 
1259
1324
  def test_yylex_identifier
@@ -1525,8 +1590,10 @@ class TestRubyLexer < Minitest::Test
1525
1590
 
1526
1591
  assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1527
1592
  s(:iter,
1528
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
1529
- s(:args, :c, :d)),
1593
+ s(:call, nil, :f,
1594
+ s(:lit, :a).line(1),
1595
+ s(:array, s(:lit, :b).line(1)).line(1)).line(1),
1596
+ s(:args, :c, :d).line(1)).line(1),
1530
1597
 
1531
1598
  :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1532
1599
  :tSYMBOL, "a", EXPR_LIT, 0, 0,
@@ -1581,7 +1648,7 @@ class TestRubyLexer < Minitest::Test
1581
1648
  s(:iter, s(:lambda),
1582
1649
  s(:args, :a)),
1583
1650
 
1584
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1651
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1585
1652
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1586
1653
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1587
1654
  :tRPAREN, ")", EXPR_ENDFN, 0, 0,
@@ -1596,7 +1663,7 @@ class TestRubyLexer < Minitest::Test
1596
1663
  s(:iter, s(:lambda),
1597
1664
  s(:args, :a)),
1598
1665
 
1599
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1666
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1600
1667
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1601
1668
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1602
1669
  :tRPAREN, ")", EXPR_ENDFN, 0, 0,
@@ -1609,7 +1676,7 @@ class TestRubyLexer < Minitest::Test
1609
1676
  s(:iter, s(:lambda),
1610
1677
  s(:args, s(:lasgn, :a, s(:nil)))),
1611
1678
 
1612
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1679
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1613
1680
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1614
1681
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1615
1682
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1626,7 +1693,7 @@ class TestRubyLexer < Minitest::Test
1626
1693
  s(:iter, s(:lambda),
1627
1694
  s(:args, s(:lasgn, :a, s(:nil)))),
1628
1695
 
1629
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1696
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1630
1697
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1631
1698
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1632
1699
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1640,7 +1707,7 @@ class TestRubyLexer < Minitest::Test
1640
1707
  assert_lex3("a -> do end do end",
1641
1708
  nil,
1642
1709
  :tIDENTIFIER, "a", EXPR_CMDARG,
1643
- :tLAMBDA, nil, EXPR_ENDFN,
1710
+ :tLAMBDA, "->", EXPR_ENDFN,
1644
1711
  :kDO, "do", EXPR_BEG,
1645
1712
  :kEND, "end", EXPR_END,
1646
1713
  :kDO, "do", EXPR_BEG,
@@ -1652,7 +1719,7 @@ class TestRubyLexer < Minitest::Test
1652
1719
  s(:iter, s(:lambda),
1653
1720
  s(:args, s(:lasgn, :a, s(:hash)))),
1654
1721
 
1655
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1722
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1656
1723
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1657
1724
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1658
1725
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1670,7 +1737,7 @@ class TestRubyLexer < Minitest::Test
1670
1737
  s(:iter, s(:lambda),
1671
1738
  s(:args, s(:lasgn, :a, s(:hash)))),
1672
1739
 
1673
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1740
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1674
1741
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1675
1742
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1676
1743
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1789,14 +1856,14 @@ class TestRubyLexer < Minitest::Test
1789
1856
  s(:defn, :"!@", s(:args), s(:nil)),
1790
1857
 
1791
1858
  :kDEF, "def", EXPR_FNAME, 0, 0,
1792
- :tUBANG, "!@", EXPR_ARG, 0, 0,
1859
+ :tBANG, "!@", EXPR_ARG, 0, 0,
1793
1860
  :tSEMI, ";", EXPR_BEG, 0, 0,
1794
1861
  :kEND, "end", EXPR_END, 0, 0)
1795
1862
  end
1796
1863
 
1797
1864
  def test_yylex_not_at_ivar
1798
1865
  assert_lex("!@ivar",
1799
- s(:call, s(:ivar, :@ivar), :"!"),
1866
+ s(:call, s(:ivar, :@ivar).line(1), :"!").line(1),
1800
1867
 
1801
1868
  :tBANG, "!", EXPR_BEG, 0, 0,
1802
1869
  :tIVAR, "@ivar", EXPR_END, 0, 0)
@@ -1805,7 +1872,7 @@ class TestRubyLexer < Minitest::Test
1805
1872
  def test_yylex_not_unary_method
1806
1873
  self.lex_state = EXPR_FNAME
1807
1874
 
1808
- assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
1875
+ assert_lex3("!@", nil, :tBANG, "!@", EXPR_ARG)
1809
1876
  end
1810
1877
 
1811
1878
  def test_yylex_nth_ref
@@ -1979,7 +2046,7 @@ class TestRubyLexer < Minitest::Test
1979
2046
 
1980
2047
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
1981
2048
  assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
1982
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2049
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
1983
2050
 
1984
2051
  emulate_string_interpolation do
1985
2052
  assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
@@ -2005,13 +2072,13 @@ class TestRubyLexer < Minitest::Test
2005
2072
 
2006
2073
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2007
2074
  assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2008
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2075
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2009
2076
 
2010
2077
  emulate_string_interpolation do
2011
2078
  assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2012
2079
  end
2013
2080
 
2014
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2081
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2015
2082
 
2016
2083
  emulate_string_interpolation do
2017
2084
  assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
@@ -2030,7 +2097,7 @@ class TestRubyLexer < Minitest::Test
2030
2097
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2031
2098
  assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2032
2099
 
2033
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2100
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2034
2101
 
2035
2102
  emulate_string_interpolation do
2036
2103
  assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
@@ -2180,7 +2247,8 @@ class TestRubyLexer < Minitest::Test
2180
2247
  end
2181
2248
 
2182
2249
  def test_yylex_regexp_escape_C_M_craaaazy
2183
- assert_lex3("/regex\\C-\\\n\\M-x/",
2250
+ rb = "/regex\\C-\\\n\\M-x/"
2251
+ assert_lex3(rb,
2184
2252
  nil,
2185
2253
  :tREGEXP_BEG, "/", EXPR_BEG,
2186
2254
  :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
@@ -2242,39 +2310,43 @@ class TestRubyLexer < Minitest::Test
2242
2310
  def test_yylex_regexp_escape_backslash_slash
2243
2311
  assert_lex3("/\\//",
2244
2312
  nil,
2245
- :tREGEXP_BEG, "/", EXPR_BEG,
2246
- :tSTRING_CONTENT, "\\/", EXPR_BEG,
2247
- :tREGEXP_END, "", EXPR_LIT)
2313
+ :tREGEXP_BEG, "/", EXPR_BEG,
2314
+ :tSTRING_CONTENT, "/", EXPR_BEG,
2315
+ :tREGEXP_END, "", EXPR_LIT)
2248
2316
  end
2249
2317
 
2250
2318
  def test_yylex_regexp_escape_backslash_terminator
2251
- assert_lex3("%r%blah\\%blah%",
2252
- nil,
2253
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2254
- :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2255
- :tREGEXP_END, "", EXPR_LIT)
2319
+ rb = "%r%blah\\%blah%"
2320
+ assert_lex3(rb,
2321
+ s(:lit, /blah%blah/).line(1),
2322
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2323
+ :tSTRING_CONTENT, "blah%blah", EXPR_BEG,
2324
+ :tREGEXP_END, "", EXPR_LIT)
2256
2325
  end
2257
2326
 
2258
2327
  def test_yylex_regexp_escape_backslash_terminator_meta1
2259
2328
  assert_lex3("%r{blah\\}blah}",
2260
- nil,
2261
- :tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
2329
+ s(:lit, /blah\}blah/).line(1),
2330
+ :tREGEXP_BEG, "%r{", EXPR_BEG,
2262
2331
  :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2263
2332
  :tREGEXP_END, "", EXPR_LIT)
2264
2333
  end
2265
2334
 
2266
2335
  def test_yylex_regexp_escape_backslash_terminator_meta2
2267
- assert_lex3("%r/blah\\/blah/",
2268
- nil,
2269
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2270
- :tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
2271
- :tREGEXP_END, "", EXPR_LIT)
2336
+ rb = "%r/blah\\/blah/"
2337
+ pt = s(:lit, /blah\/blah/).line 1
2338
+
2339
+ assert_lex3(rb,
2340
+ pt,
2341
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2342
+ :tSTRING_CONTENT, "blah/blah", EXPR_BEG,
2343
+ :tREGEXP_END, "", EXPR_LIT)
2272
2344
  end
2273
2345
 
2274
2346
  def test_yylex_regexp_escape_backslash_terminator_meta3
2275
2347
  assert_lex3("%r/blah\\%blah/",
2276
2348
  nil,
2277
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2349
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2278
2350
  :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2279
2351
  :tREGEXP_END, "", EXPR_LIT)
2280
2352
  end
@@ -2284,8 +2356,9 @@ class TestRubyLexer < Minitest::Test
2284
2356
  end
2285
2357
 
2286
2358
  def test_yylex_regexp_escape_bs
2287
- assert_lex3("/regex\\\\regex/",
2288
- nil,
2359
+ rp = "/regex\\\\regex/"
2360
+ assert_lex3(rp,
2361
+ s(:lit, /regex\\regex/),
2289
2362
  :tREGEXP_BEG, "/", EXPR_BEG,
2290
2363
  :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2291
2364
  :tREGEXP_END, "", EXPR_LIT)
@@ -2316,12 +2389,14 @@ class TestRubyLexer < Minitest::Test
2316
2389
  end
2317
2390
 
2318
2391
  def test_yylex_regexp_escape_double_backslash
2319
- regexp = '/[\\/\\\\]$/'
2320
- assert_lex3(regexp.dup,
2321
- nil,
2322
- :tREGEXP_BEG, "/", EXPR_BEG,
2323
- :tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
2324
- :tREGEXP_END, "", EXPR_LIT)
2392
+ rb = '/[\\/\\\\]$/'
2393
+ pt = s(:lit, /[\/\\]$/)
2394
+
2395
+ assert_lex3(rb,
2396
+ pt,
2397
+ :tREGEXP_BEG, "/", EXPR_BEG,
2398
+ :tSTRING_CONTENT, "[/\\\\]$", EXPR_BEG,
2399
+ :tREGEXP_END, "", EXPR_LIT)
2325
2400
  end
2326
2401
 
2327
2402
  def test_yylex_regexp_escape_hex
@@ -2379,7 +2454,7 @@ class TestRubyLexer < Minitest::Test
2379
2454
  def test_yylex_regexp_escaped_delim
2380
2455
  assert_lex3("%r!blah(?\\!blah)!",
2381
2456
  nil,
2382
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2457
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2383
2458
  :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2384
2459
  :tREGEXP_END, "", EXPR_LIT)
2385
2460
  end
@@ -2494,7 +2569,9 @@ class TestRubyLexer < Minitest::Test
2494
2569
  end
2495
2570
 
2496
2571
  def test_yylex_string_bad_eos_quote
2497
- refute_lex("%{nest", :tSTRING_BEG, "%}")
2572
+ refute_lex("%{nest",
2573
+ :tSTRING_BEG, "%}",
2574
+ :tSTRING_CONTENT, "nest")
2498
2575
  end
2499
2576
 
2500
2577
  def test_yylex_string_double
@@ -2506,7 +2583,7 @@ class TestRubyLexer < Minitest::Test
2506
2583
  end
2507
2584
 
2508
2585
  def test_yylex_string_double_escape_C_backslash
2509
- assert_lex3("\"\\C-\\\\\"",
2586
+ assert_lex3(%W[ " \\ C - \\ \\ " ].join, # I hate escaping \ in ' and "
2510
2587
  nil,
2511
2588
  :tSTRING_BEG, "\"", EXPR_BEG,
2512
2589
  :tSTRING_CONTENT, "\034", EXPR_BEG,
@@ -2560,7 +2637,9 @@ class TestRubyLexer < Minitest::Test
2560
2637
  end
2561
2638
 
2562
2639
  def test_yylex_string_double_escape_c_backslash
2563
- refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
2640
+ refute_lex('"\\c\\"',
2641
+ :tSTRING_BEG, '"',
2642
+ :tSTRING_CONTENT, "\002")
2564
2643
  end
2565
2644
 
2566
2645
  def test_yylex_string_double_escape_c_escape
@@ -2596,12 +2675,12 @@ class TestRubyLexer < Minitest::Test
2596
2675
  nil,
2597
2676
  :tSTRING_BEG, "\"", EXPR_BEG,
2598
2677
  :tSTRING_CONTENT, "blah #x a ", EXPR_BEG,
2599
- :tSTRING_DVAR, nil, EXPR_BEG,
2678
+ :tSTRING_DVAR, "#", EXPR_BEG,
2600
2679
  :tSTRING_CONTENT, "@a b ", EXPR_BEG,
2601
- :tSTRING_DVAR, nil, EXPR_BEG,
2680
+ :tSTRING_DVAR, "#", EXPR_BEG,
2602
2681
  :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2603
- :tSTRING_DBEG, nil, EXPR_BEG,
2604
- :tSTRING_CONTENT, "3} # ", EXPR_BEG,
2682
+ :tSTRING_DBEG, "#\{", EXPR_BEG,
2683
+ :tSTRING_CONTENT, "3} # ", EXPR_BEG, # FIX: wrong!?!?
2605
2684
  :tSTRING_END, "\"", EXPR_LIT)
2606
2685
  end
2607
2686
 
@@ -2635,12 +2714,12 @@ class TestRubyLexer < Minitest::Test
2635
2714
  nil,
2636
2715
  :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2637
2716
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2638
- :tSPACE, nil, EXPR_BEG,
2717
+ :tSPACE, " ", EXPR_BEG,
2639
2718
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2640
- :tSPACE, nil, EXPR_BEG,
2719
+ :tSPACE, " ", EXPR_BEG,
2641
2720
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2642
- :tSPACE, nil, EXPR_BEG,
2643
- :tSTRING_END, nil, EXPR_LIT)
2721
+ :tSPACE, "]", EXPR_BEG,
2722
+ :tSTRING_END, "]", EXPR_LIT)
2644
2723
  end
2645
2724
 
2646
2725
  def test_yylex_string_pct_I_extra_space
@@ -2648,12 +2727,12 @@ class TestRubyLexer < Minitest::Test
2648
2727
  nil,
2649
2728
  :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2650
2729
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2651
- :tSPACE, nil, EXPR_BEG,
2730
+ :tSPACE, " ", EXPR_BEG,
2652
2731
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2653
- :tSPACE, nil, EXPR_BEG,
2732
+ :tSPACE, " ", EXPR_BEG,
2654
2733
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2655
- :tSPACE, nil, EXPR_BEG,
2656
- :tSTRING_END, nil, EXPR_LIT)
2734
+ :tSPACE, "]", EXPR_BEG,
2735
+ :tSTRING_END, "]", EXPR_LIT)
2657
2736
  end
2658
2737
 
2659
2738
  def test_yylex_string_pct_Q
@@ -2685,23 +2764,28 @@ class TestRubyLexer < Minitest::Test
2685
2764
  nil,
2686
2765
  :tWORDS_BEG, "%W[", EXPR_BEG,
2687
2766
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2688
- :tSPACE, nil, EXPR_BEG,
2767
+ :tSPACE, " ", EXPR_BEG,
2689
2768
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2690
- :tSPACE, nil, EXPR_BEG,
2769
+ :tSPACE, " ", EXPR_BEG,
2691
2770
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2692
- :tSPACE, nil, EXPR_BEG,
2693
- :tSTRING_END, nil, EXPR_LIT)
2771
+ :tSPACE, "]", EXPR_BEG,
2772
+ :tSTRING_END, "]", EXPR_LIT)
2694
2773
  end
2695
2774
 
2696
2775
  def test_yylex_string_pct_W_bs_nl
2697
- assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2698
- nil,
2776
+ rb = "%W[s1 \\\ns2]" # TODO: add interpolation to these
2777
+ pt = s(:array,
2778
+ s(:str, "s1").line(1),
2779
+ s(:str, "\ns2").line(1)).line(1)
2780
+
2781
+ assert_lex3(rb,
2782
+ pt,
2699
2783
  :tWORDS_BEG, "%W[", EXPR_BEG,
2700
2784
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2701
- :tSPACE, nil, EXPR_BEG,
2785
+ :tSPACE, " ", EXPR_BEG,
2702
2786
  :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2703
- :tSPACE, nil, EXPR_BEG,
2704
- :tSTRING_END, nil, EXPR_LIT)
2787
+ :tSPACE, "]", EXPR_BEG,
2788
+ :tSTRING_END, "]", EXPR_LIT)
2705
2789
  end
2706
2790
 
2707
2791
  def test_yylex_string_pct_angle
@@ -2717,12 +2801,12 @@ class TestRubyLexer < Minitest::Test
2717
2801
  nil,
2718
2802
  :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2719
2803
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2720
- :tSPACE, nil, EXPR_BEG,
2804
+ :tSPACE, " ", EXPR_BEG,
2721
2805
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2722
- :tSPACE, nil, EXPR_BEG,
2806
+ :tSPACE, " ", EXPR_BEG,
2723
2807
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2724
- :tSPACE, nil, EXPR_BEG,
2725
- :tSTRING_END, nil, EXPR_LIT)
2808
+ :tSPACE, "]", EXPR_BEG,
2809
+ :tSTRING_END, "]", EXPR_LIT)
2726
2810
  end
2727
2811
 
2728
2812
  def test_yylex_string_pct_i_extra_space
@@ -2730,12 +2814,12 @@ class TestRubyLexer < Minitest::Test
2730
2814
  nil,
2731
2815
  :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2732
2816
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2733
- :tSPACE, nil, EXPR_BEG,
2817
+ :tSPACE, " ", EXPR_BEG,
2734
2818
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2735
- :tSPACE, nil, EXPR_BEG,
2819
+ :tSPACE, " ", EXPR_BEG,
2736
2820
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2737
- :tSPACE, nil, EXPR_BEG,
2738
- :tSTRING_END, nil, EXPR_LIT)
2821
+ :tSPACE, "]", EXPR_BEG,
2822
+ :tSTRING_END, "]", EXPR_LIT)
2739
2823
  end
2740
2824
 
2741
2825
  def test_yylex_string_pct_other
@@ -2766,9 +2850,9 @@ class TestRubyLexer < Minitest::Test
2766
2850
  refute_lex("%w[s1 s2 ",
2767
2851
  :tQWORDS_BEG, "%w[",
2768
2852
  :tSTRING_CONTENT, "s1",
2769
- :tSPACE, nil,
2853
+ :tSPACE, " ",
2770
2854
  :tSTRING_CONTENT, "s2",
2771
- :tSPACE, nil)
2855
+ :tSPACE, " ")
2772
2856
  end
2773
2857
 
2774
2858
  def test_yylex_string_pct_w_bs_nl
@@ -2776,21 +2860,21 @@ class TestRubyLexer < Minitest::Test
2776
2860
  nil,
2777
2861
  :tQWORDS_BEG, "%w[", EXPR_BEG,
2778
2862
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2779
- :tSPACE, nil, EXPR_BEG,
2863
+ :tSPACE, " ", EXPR_BEG,
2780
2864
  :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2781
- :tSPACE, nil, EXPR_BEG,
2782
- :tSTRING_END, nil, EXPR_LIT)
2865
+ :tSPACE, "]", EXPR_BEG,
2866
+ :tSTRING_END, "]", EXPR_LIT)
2783
2867
  end
2784
2868
 
2785
2869
  def test_yylex_string_pct_w_bs_sp
2786
2870
  assert_lex3("%w[s\\ 1 s\\ 2]",
2787
- nil,
2871
+ s(:array, s(:str, "s 1"), s(:str, "s 2")),
2788
2872
  :tQWORDS_BEG, "%w[", EXPR_BEG,
2789
2873
  :tSTRING_CONTENT, "s 1", EXPR_BEG,
2790
- :tSPACE, nil, EXPR_BEG,
2874
+ :tSPACE, " ", EXPR_BEG,
2791
2875
  :tSTRING_CONTENT, "s 2", EXPR_BEG,
2792
- :tSPACE, nil, EXPR_BEG,
2793
- :tSTRING_END, nil, EXPR_LIT)
2876
+ :tSPACE, "]", EXPR_BEG,
2877
+ :tSTRING_END, "]", EXPR_LIT)
2794
2878
  end
2795
2879
 
2796
2880
  def test_yylex_string_single
@@ -2820,7 +2904,7 @@ class TestRubyLexer < Minitest::Test
2820
2904
  assert_lex3('"#@a\u{3024}"',
2821
2905
  s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2822
2906
  :tSTRING_BEG, '"', EXPR_BEG,
2823
- :tSTRING_DVAR, nil, EXPR_BEG,
2907
+ :tSTRING_DVAR, "#", EXPR_BEG,
2824
2908
  :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2825
2909
  :tSTRING_END, '"', EXPR_LIT)
2826
2910
  end
@@ -2831,7 +2915,7 @@ class TestRubyLexer < Minitest::Test
2831
2915
 
2832
2916
  refute_lex('"#@a\u302zzz"',
2833
2917
  :tSTRING_BEG, '"',
2834
- :tSTRING_DVAR, nil,
2918
+ :tSTRING_DVAR, "#",
2835
2919
  :tSTRING_CONTENT, "@a"+str,
2836
2920
  :tSTRING_END, '"')
2837
2921
 
@@ -2840,7 +2924,7 @@ class TestRubyLexer < Minitest::Test
2840
2924
 
2841
2925
  refute_lex('"#@a\u30zzz"',
2842
2926
  :tSTRING_BEG, '"',
2843
- :tSTRING_DVAR, nil,
2927
+ :tSTRING_DVAR, "#",
2844
2928
  :tSTRING_CONTENT, "@a"+str,
2845
2929
  :tSTRING_END, '"')
2846
2930
 
@@ -2849,7 +2933,7 @@ class TestRubyLexer < Minitest::Test
2849
2933
 
2850
2934
  refute_lex('"#@a\u3zzz"',
2851
2935
  :tSTRING_BEG, '"',
2852
- :tSTRING_DVAR, nil,
2936
+ :tSTRING_DVAR, "#",
2853
2937
  :tSTRING_CONTENT, "@a"+str,
2854
2938
  :tSTRING_END, '"')
2855
2939
  end
@@ -2870,7 +2954,7 @@ class TestRubyLexer < Minitest::Test
2870
2954
  assert_lex3('"#@a\u3024abz"',
2871
2955
  s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2872
2956
  :tSTRING_BEG, '"', EXPR_BEG,
2873
- :tSTRING_DVAR, nil, EXPR_BEG,
2957
+ :tSTRING_DVAR, "#", EXPR_BEG,
2874
2958
  :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2875
2959
  :tSTRING_END, '"', EXPR_LIT)
2876
2960
  end
@@ -2920,7 +3004,7 @@ class TestRubyLexer < Minitest::Test
2920
3004
  nil,
2921
3005
  :tSYMBEG, ":", EXPR_FNAME,
2922
3006
  :tSTRING_CONTENT, "symbol", EXPR_FNAME,
2923
- :tSTRING_DBEG, nil, EXPR_FNAME,
3007
+ :tSTRING_DBEG, '#{', EXPR_FNAME,
2924
3008
  :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
2925
3009
  :tSTRING_END, "\"", EXPR_LIT)
2926
3010
  end