ruby_parser 3.17.0 → 3.18.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/History.rdoc +76 -0
  4. data/Manifest.txt +3 -0
  5. data/README.rdoc +1 -0
  6. data/Rakefile +68 -18
  7. data/bin/ruby_parse_extract_error +1 -1
  8. data/compare/normalize.rb +6 -1
  9. data/gauntlet.md +106 -0
  10. data/lib/rp_extensions.rb +15 -36
  11. data/lib/rp_stringscanner.rb +20 -51
  12. data/lib/ruby20_parser.rb +3445 -3394
  13. data/lib/ruby20_parser.y +326 -248
  14. data/lib/ruby21_parser.rb +3543 -3511
  15. data/lib/ruby21_parser.y +321 -245
  16. data/lib/ruby22_parser.rb +3553 -3512
  17. data/lib/ruby22_parser.y +325 -247
  18. data/lib/ruby23_parser.rb +3566 -3530
  19. data/lib/ruby23_parser.y +325 -247
  20. data/lib/ruby24_parser.rb +3595 -3548
  21. data/lib/ruby24_parser.y +325 -247
  22. data/lib/ruby25_parser.rb +3595 -3547
  23. data/lib/ruby25_parser.y +325 -247
  24. data/lib/ruby26_parser.rb +3605 -3560
  25. data/lib/ruby26_parser.y +324 -246
  26. data/lib/ruby27_parser.rb +4657 -3539
  27. data/lib/ruby27_parser.y +878 -253
  28. data/lib/ruby30_parser.rb +5230 -3882
  29. data/lib/ruby30_parser.y +1069 -321
  30. data/lib/ruby3_parser.yy +3467 -0
  31. data/lib/ruby_lexer.rb +261 -609
  32. data/lib/ruby_lexer.rex +27 -20
  33. data/lib/ruby_lexer.rex.rb +59 -23
  34. data/lib/ruby_lexer_strings.rb +638 -0
  35. data/lib/ruby_parser.yy +910 -263
  36. data/lib/ruby_parser_extras.rb +289 -114
  37. data/test/test_ruby_lexer.rb +181 -129
  38. data/test/test_ruby_parser.rb +1213 -108
  39. data/tools/munge.rb +34 -6
  40. data/tools/ripper.rb +15 -10
  41. data.tar.gz.sig +0 -0
  42. metadata +11 -12
  43. metadata.gz.sig +0 -0
data/test/test_ruby_lexer.rb CHANGED
@@ -46,11 +46,21 @@ class TestRubyLexer < Minitest::Test
46
46
  end
47
47
 
48
48
  def assert_lex3 input, exp_sexp, *args, &block
49
+ # TODO: refute_nil exp_sexp, "Get off your lazy butt and write one"
50
+
49
51
  args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
50
52
 
51
53
  assert_lex(input, exp_sexp, *args, &block)
52
54
  end
53
55
 
56
+ def refute_lex3 input, *args # TODO: re-sort
57
+ args = args.each_slice(3).map { |a, b, c| [a, b, c, nil, nil] }.flatten
58
+
59
+ assert_raises RubyParser::SyntaxError do
60
+ assert_lex(input, nil, *args)
61
+ end
62
+ end
63
+
54
64
  def assert_lex_fname name, type, end_state = EXPR_ARG # TODO: swap name/type
55
65
  assert_lex3("def #{name} ",
56
66
  nil,
@@ -95,15 +105,15 @@ class TestRubyLexer < Minitest::Test
95
105
  end
96
106
 
97
107
  def assert_read_escape expected, input
98
- @lex.ss.string = input.dup
108
+ setup_lexer input
99
109
  enc = expected.encoding
100
- assert_equal expected, @lex.read_escape.b.force_encoding(enc), input
110
+ assert_equal expected, lex.read_escape.b.force_encoding(enc), input
101
111
  end
102
112
 
103
113
  def assert_read_escape_bad input # TODO: rename refute_read_escape
104
- @lex.ss.string = input
114
+ setup_lexer input
105
115
  assert_raises RubyParser::SyntaxError do
106
- @lex.read_escape
116
+ lex.read_escape
107
117
  end
108
118
  end
109
119
 
@@ -124,7 +134,7 @@ class TestRubyLexer < Minitest::Test
124
134
  def refute_lexeme
125
135
  x = y = @lex.next_token
126
136
 
127
- refute x, "not empty: #{y.inspect}"
137
+ refute x, "not empty: #{y.inspect}: #{@lex.rest.inspect}"
128
138
  end
129
139
 
130
140
  ## Utility Methods:
@@ -159,8 +169,8 @@ class TestRubyLexer < Minitest::Test
159
169
  ## Tests:
160
170
 
161
171
  def test_next_token
162
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
163
- assert_equal [:tIDENTIFIER, "blah"], @lex.next_token
172
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
173
+ assert_equal [:tIDENTIFIER, ["blah", 1]], @lex.next_token
164
174
  assert_nil @lex.next_token
165
175
  end
166
176
 
@@ -173,10 +183,10 @@ class TestRubyLexer < Minitest::Test
173
183
 
174
184
  :tQWORDS_BEG, "%w[", EXPR_BEG, 0, 0,
175
185
  :tSTRING_CONTENT, "foo", EXPR_BEG, 0, 0,
176
- :tSPACE, nil, EXPR_BEG, 0, 0,
186
+ :tSPACE, " ", EXPR_BEG, 0, 0,
177
187
  :tSTRING_CONTENT, "bar", EXPR_BEG, 0, 0,
178
- :tSPACE, nil, EXPR_BEG, 0, 0,
179
- :tSTRING_END, nil, EXPR_LIT, 0, 0)
188
+ :tSPACE, "]", EXPR_BEG, 0, 0,
189
+ :tSTRING_END, "]", EXPR_LIT, 0, 0)
180
190
  end
181
191
  end
182
192
 
@@ -686,8 +696,8 @@ class TestRubyLexer < Minitest::Test
686
696
  end
687
697
 
688
698
  def test_yylex_def_bad_name
689
- self.lex_state = EXPR_FNAME
690
- refute_lex("def [ ", :kDEF, "def")
699
+ refute_lex3("def [ ",
700
+ :kDEF, "def", EXPR_FNAME)
691
701
  end
692
702
 
693
703
  def test_yylex_div
@@ -756,10 +766,6 @@ class TestRubyLexer < Minitest::Test
756
766
  assert_includes(e.message, "is not allowed as a global variable name")
757
767
  end
758
768
 
759
- def test_yylex_dollar_eos
760
- assert_lex3("$", nil, "$", "$", EXPR_END) # FIX: wtf is this?!?
761
- end
762
-
763
769
  def test_yylex_dot # HINT message sends
764
770
  assert_lex3(".", nil, :tDOT, ".", EXPR_DOT)
765
771
  end
@@ -938,6 +944,9 @@ class TestRubyLexer < Minitest::Test
938
944
  end
939
945
 
940
946
  def test_yylex_global_dash_nothing
947
+ refute_lex3("$- ", nil) # fails 2.1+
948
+
949
+ setup_lexer_class RubyParser::V20
941
950
  assert_lex3("$- ", nil, :tGVAR, "$-", EXPR_END)
942
951
  end
943
952
 
@@ -1095,7 +1104,7 @@ class TestRubyLexer < Minitest::Test
1095
1104
  end
1096
1105
 
1097
1106
  def test_yylex_heredoc_backtick
1098
- assert_lex3("a = <<`EOF`\n blah blah\nEOF\n\n",
1107
+ assert_lex3("a = <<`EOF`\n blah blah\nEOF\n",
1099
1108
  nil,
1100
1109
  :tIDENTIFIER, "a", EXPR_CMDARG,
1101
1110
  :tEQL, "=", EXPR_BEG,
@@ -1117,21 +1126,41 @@ class TestRubyLexer < Minitest::Test
1117
1126
  end
1118
1127
 
1119
1128
  def test_yylex_heredoc_double_dash
1120
- assert_lex3("a = <<-\"EOF\"\n blah blah\n EOF\n\n",
1129
+ assert_lex3("a = \" blah blah\n\".strip\n42",
1130
+ nil,
1131
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1132
+ :tEQL, "=", EXPR_BEG,
1133
+ :tSTRING, " blah blah\n", EXPR_END,
1134
+ :tDOT, ".", EXPR_DOT,
1135
+ :tIDENTIFIER, "strip", EXPR_ARG,
1136
+ :tNL, nil, EXPR_BEG,
1137
+
1138
+ :tINTEGER, 42, EXPR_END
1139
+ )
1140
+
1141
+ assert_lex3("a = <<-\"EOF\".strip\n blah blah\n EOF\n42",
1121
1142
  nil,
1122
1143
  :tIDENTIFIER, "a", EXPR_CMDARG,
1123
1144
  :tEQL, "=", EXPR_BEG,
1124
1145
  :tSTRING_BEG, "\"", EXPR_BEG,
1125
1146
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1126
1147
  :tSTRING_END, "EOF", EXPR_LIT,
1127
- :tNL, nil, EXPR_BEG)
1148
+
1149
+ :tDOT, ".", EXPR_DOT,
1150
+ :tIDENTIFIER, "strip", EXPR_ARG,
1151
+
1152
+ :tNL, nil, EXPR_BEG,
1153
+
1154
+ :tINTEGER, 42, EXPR_END
1155
+ )
1128
1156
  end
1129
1157
 
1130
1158
  def test_yylex_heredoc_double_eos
1131
1159
  refute_lex("a = <<\"EOF\"\nblah",
1132
1160
  :tIDENTIFIER, "a",
1133
1161
  :tEQL, "=",
1134
- :tSTRING_BEG, "\"")
1162
+ :tSTRING_BEG, "\"",
1163
+ :tSTRING_CONTENT, "blah")
1135
1164
  end
1136
1165
 
1137
1166
  def test_yylex_heredoc_double_eos_nl
@@ -1165,12 +1194,12 @@ class TestRubyLexer < Minitest::Test
1165
1194
 
1166
1195
  assert_lex3("a = <<~\"EOF\"\n blah blah\n EOF\n\n",
1167
1196
  nil,
1168
- :tIDENTIFIER, "a", EXPR_CMDARG,
1169
- :tEQL, "=", EXPR_BEG,
1170
- :tSTRING_BEG, "\"", EXPR_BEG,
1197
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1198
+ :tEQL, "=", EXPR_BEG,
1199
+ :tSTRING_BEG, "\"", EXPR_BEG,
1171
1200
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1172
- :tSTRING_END, "EOF", EXPR_LIT,
1173
- :tNL, nil, EXPR_BEG)
1201
+ :tSTRING_END, "EOF", EXPR_LIT,
1202
+ :tNL, nil, EXPR_BEG)
1174
1203
  end
1175
1204
 
1176
1205
  def test_yylex_heredoc_empty
@@ -1217,37 +1246,41 @@ class TestRubyLexer < Minitest::Test
1217
1246
 
1218
1247
  assert_lex3("a = <<~EOF\n blah\n blah\n EOF\n",
1219
1248
  nil,
1220
- :tIDENTIFIER, "a", EXPR_CMDARG,
1221
- :tEQL, "=", EXPR_BEG,
1222
- :tSTRING_BEG, "\"", EXPR_BEG,
1249
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1250
+ :tEQL, "=", EXPR_BEG,
1251
+ :tSTRING_BEG, "\"", EXPR_BEG,
1223
1252
  :tSTRING_CONTENT, " blah\n blah\n", EXPR_BEG,
1224
- :tSTRING_END, "EOF", EXPR_LIT,
1225
- :tNL, nil, EXPR_BEG)
1253
+ :tSTRING_END, "EOF", EXPR_LIT,
1254
+ :tNL, nil, EXPR_BEG)
1226
1255
  end
1227
1256
 
1228
1257
  def test_yylex_heredoc_single
1229
- assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n",
1258
+ assert_lex3("a = <<'EOF'\n blah blah\nEOF\n\n\n\n42\n",
1230
1259
  nil,
1231
1260
  :tIDENTIFIER, "a", EXPR_CMDARG,
1232
1261
  :tEQL, "=", EXPR_BEG,
1233
- :tSTRING_BEG, "\"", EXPR_BEG,
1262
+ :tSTRING_BEG, "'", EXPR_BEG,
1234
1263
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1235
1264
  :tSTRING_END, "EOF", EXPR_LIT,
1265
+ :tNL, nil, EXPR_BEG,
1266
+ :tINTEGER, 42, EXPR_LIT,
1236
1267
  :tNL, nil, EXPR_BEG)
1268
+
1269
+ assert_nil lex.old_ss
1237
1270
  end
1238
1271
 
1239
1272
  def test_yylex_heredoc_single_bad_eos_body
1240
1273
  refute_lex("a = <<'EOF'\nblah",
1241
1274
  :tIDENTIFIER, "a",
1242
1275
  :tEQL, "=",
1243
- :tSTRING_BEG, "\"")
1276
+ :tSTRING_BEG, "'")
1244
1277
  end
1245
1278
 
1246
1279
  def test_yylex_heredoc_single_bad_eos_empty
1247
1280
  refute_lex("a = <<''\n",
1248
1281
  :tIDENTIFIER, "a",
1249
1282
  :tEQL, "=",
1250
- :tSTRING_BEG, "\"")
1283
+ :tSTRING_BEG, "'")
1251
1284
  end
1252
1285
 
1253
1286
  def test_yylex_heredoc_single_bad_eos_term
@@ -1269,7 +1302,7 @@ class TestRubyLexer < Minitest::Test
1269
1302
  nil,
1270
1303
  :tIDENTIFIER, "a", EXPR_CMDARG,
1271
1304
  :tEQL, "=", EXPR_BEG,
1272
- :tSTRING_BEG, "\"", EXPR_BEG,
1305
+ :tSTRING_BEG, "'", EXPR_BEG,
1273
1306
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1274
1307
  :tSTRING_END, "EOF", EXPR_LIT,
1275
1308
  :tNL, nil, EXPR_BEG)
@@ -1280,12 +1313,12 @@ class TestRubyLexer < Minitest::Test
1280
1313
 
1281
1314
  assert_lex3("a = <<~'EOF'\n blah blah\n EOF\n\n",
1282
1315
  nil,
1283
- :tIDENTIFIER, "a", EXPR_CMDARG,
1284
- :tEQL, "=", EXPR_BEG,
1285
- :tSTRING_BEG, "\"", EXPR_BEG,
1316
+ :tIDENTIFIER, "a", EXPR_CMDARG,
1317
+ :tEQL, "=", EXPR_BEG,
1318
+ :tSTRING_BEG, "'", EXPR_BEG,
1286
1319
  :tSTRING_CONTENT, " blah blah\n", EXPR_BEG,
1287
- :tSTRING_END, "EOF", EXPR_LIT,
1288
- :tNL, nil, EXPR_BEG)
1320
+ :tSTRING_END, "EOF", EXPR_LIT,
1321
+ :tNL, nil, EXPR_BEG)
1289
1322
  end
1290
1323
 
1291
1324
  def test_yylex_identifier
@@ -1557,8 +1590,10 @@ class TestRubyLexer < Minitest::Test
1557
1590
 
1558
1591
  assert_lex("f :a, [:b] { |c, d| }", # yes, this is bad code
1559
1592
  s(:iter,
1560
- s(:call, nil, :f, s(:lit, :a), s(:array, s(:lit, :b))),
1561
- s(:args, :c, :d)),
1593
+ s(:call, nil, :f,
1594
+ s(:lit, :a).line(1),
1595
+ s(:array, s(:lit, :b).line(1)).line(1)).line(1),
1596
+ s(:args, :c, :d).line(1)).line(1),
1562
1597
 
1563
1598
  :tIDENTIFIER, "f", EXPR_CMDARG, 0, 0,
1564
1599
  :tSYMBOL, "a", EXPR_LIT, 0, 0,
@@ -1613,7 +1648,7 @@ class TestRubyLexer < Minitest::Test
1613
1648
  s(:iter, s(:lambda),
1614
1649
  s(:args, :a)),
1615
1650
 
1616
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1651
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1617
1652
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1618
1653
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1619
1654
  :tRPAREN, ")", EXPR_ENDFN, 0, 0,
@@ -1628,7 +1663,7 @@ class TestRubyLexer < Minitest::Test
1628
1663
  s(:iter, s(:lambda),
1629
1664
  s(:args, :a)),
1630
1665
 
1631
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1666
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1632
1667
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1633
1668
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1634
1669
  :tRPAREN, ")", EXPR_ENDFN, 0, 0,
@@ -1641,7 +1676,7 @@ class TestRubyLexer < Minitest::Test
1641
1676
  s(:iter, s(:lambda),
1642
1677
  s(:args, s(:lasgn, :a, s(:nil)))),
1643
1678
 
1644
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1679
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1645
1680
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1646
1681
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1647
1682
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1658,7 +1693,7 @@ class TestRubyLexer < Minitest::Test
1658
1693
  s(:iter, s(:lambda),
1659
1694
  s(:args, s(:lasgn, :a, s(:nil)))),
1660
1695
 
1661
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1696
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1662
1697
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1663
1698
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1664
1699
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1672,7 +1707,7 @@ class TestRubyLexer < Minitest::Test
1672
1707
  assert_lex3("a -> do end do end",
1673
1708
  nil,
1674
1709
  :tIDENTIFIER, "a", EXPR_CMDARG,
1675
- :tLAMBDA, nil, EXPR_ENDFN,
1710
+ :tLAMBDA, "->", EXPR_ENDFN,
1676
1711
  :kDO, "do", EXPR_BEG,
1677
1712
  :kEND, "end", EXPR_END,
1678
1713
  :kDO, "do", EXPR_BEG,
@@ -1684,7 +1719,7 @@ class TestRubyLexer < Minitest::Test
1684
1719
  s(:iter, s(:lambda),
1685
1720
  s(:args, s(:lasgn, :a, s(:hash)))),
1686
1721
 
1687
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1722
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1688
1723
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1689
1724
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1690
1725
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1702,7 +1737,7 @@ class TestRubyLexer < Minitest::Test
1702
1737
  s(:iter, s(:lambda),
1703
1738
  s(:args, s(:lasgn, :a, s(:hash)))),
1704
1739
 
1705
- :tLAMBDA, nil, EXPR_ENDFN, 0, 0,
1740
+ :tLAMBDA, "->", EXPR_ENDFN, 0, 0,
1706
1741
  :tLPAREN2, "(", EXPR_PAR, 1, 0,
1707
1742
  :tIDENTIFIER, "a", EXPR_ARG, 1, 0,
1708
1743
  :tEQL, "=", EXPR_BEG, 1, 0,
@@ -1821,14 +1856,14 @@ class TestRubyLexer < Minitest::Test
1821
1856
  s(:defn, :"!@", s(:args), s(:nil)),
1822
1857
 
1823
1858
  :kDEF, "def", EXPR_FNAME, 0, 0,
1824
- :tUBANG, "!@", EXPR_ARG, 0, 0,
1859
+ :tBANG, "!@", EXPR_ARG, 0, 0,
1825
1860
  :tSEMI, ";", EXPR_BEG, 0, 0,
1826
1861
  :kEND, "end", EXPR_END, 0, 0)
1827
1862
  end
1828
1863
 
1829
1864
  def test_yylex_not_at_ivar
1830
1865
  assert_lex("!@ivar",
1831
- s(:call, s(:ivar, :@ivar), :"!"),
1866
+ s(:call, s(:ivar, :@ivar).line(1), :"!").line(1),
1832
1867
 
1833
1868
  :tBANG, "!", EXPR_BEG, 0, 0,
1834
1869
  :tIVAR, "@ivar", EXPR_END, 0, 0)
@@ -1837,7 +1872,7 @@ class TestRubyLexer < Minitest::Test
1837
1872
  def test_yylex_not_unary_method
1838
1873
  self.lex_state = EXPR_FNAME
1839
1874
 
1840
- assert_lex3("!@", nil, :tUBANG, "!@", EXPR_ARG)
1875
+ assert_lex3("!@", nil, :tBANG, "!@", EXPR_ARG)
1841
1876
  end
1842
1877
 
1843
1878
  def test_yylex_nth_ref
@@ -2011,7 +2046,7 @@ class TestRubyLexer < Minitest::Test
2011
2046
 
2012
2047
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2013
2048
  assert_next_lexeme :tSTRING_CONTENT, " ", EXPR_BEG, 0, 0
2014
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2049
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2015
2050
 
2016
2051
  emulate_string_interpolation do
2017
2052
  assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
@@ -2037,13 +2072,13 @@ class TestRubyLexer < Minitest::Test
2037
2072
 
2038
2073
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2039
2074
  assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2040
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2075
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2041
2076
 
2042
2077
  emulate_string_interpolation do
2043
2078
  assert_next_lexeme :tIDENTIFIER, "b", EXPR_CMDARG, 0, 0
2044
2079
  end
2045
2080
 
2046
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2081
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2047
2082
 
2048
2083
  emulate_string_interpolation do
2049
2084
  assert_next_lexeme :tIDENTIFIER, "d", EXPR_CMDARG, 0, 0
@@ -2062,7 +2097,7 @@ class TestRubyLexer < Minitest::Test
2062
2097
  assert_next_lexeme :tSTRING_BEG, "%)", EXPR_BEG, 0, 0
2063
2098
  assert_next_lexeme :tSTRING_CONTENT, "(", EXPR_BEG, 0, 0
2064
2099
 
2065
- assert_next_lexeme :tSTRING_DBEG, nil, EXPR_BEG, 0, 0
2100
+ assert_next_lexeme :tSTRING_DBEG, '#{', EXPR_BEG, 0, 0
2066
2101
 
2067
2102
  emulate_string_interpolation do
2068
2103
  assert_next_lexeme :tLPAREN, "(", EXPR_PAR, 1, 0
@@ -2212,7 +2247,8 @@ class TestRubyLexer < Minitest::Test
2212
2247
  end
2213
2248
 
2214
2249
  def test_yylex_regexp_escape_C_M_craaaazy
2215
- assert_lex3("/regex\\C-\\\n\\M-x/",
2250
+ rb = "/regex\\C-\\\n\\M-x/"
2251
+ assert_lex3(rb,
2216
2252
  nil,
2217
2253
  :tREGEXP_BEG, "/", EXPR_BEG,
2218
2254
  :tSTRING_CONTENT, "regex\\C-\\M-x", EXPR_BEG,
@@ -2274,39 +2310,43 @@ class TestRubyLexer < Minitest::Test
2274
2310
  def test_yylex_regexp_escape_backslash_slash
2275
2311
  assert_lex3("/\\//",
2276
2312
  nil,
2277
- :tREGEXP_BEG, "/", EXPR_BEG,
2278
- :tSTRING_CONTENT, "\\/", EXPR_BEG,
2279
- :tREGEXP_END, "", EXPR_LIT)
2313
+ :tREGEXP_BEG, "/", EXPR_BEG,
2314
+ :tSTRING_CONTENT, "/", EXPR_BEG,
2315
+ :tREGEXP_END, "", EXPR_LIT)
2280
2316
  end
2281
2317
 
2282
2318
  def test_yylex_regexp_escape_backslash_terminator
2283
- assert_lex3("%r%blah\\%blah%",
2284
- nil,
2285
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2286
- :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2287
- :tREGEXP_END, "", EXPR_LIT)
2319
+ rb = "%r%blah\\%blah%"
2320
+ assert_lex3(rb,
2321
+ s(:lit, /blah%blah/).line(1),
2322
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2323
+ :tSTRING_CONTENT, "blah%blah", EXPR_BEG,
2324
+ :tREGEXP_END, "", EXPR_LIT)
2288
2325
  end
2289
2326
 
2290
2327
  def test_yylex_regexp_escape_backslash_terminator_meta1
2291
2328
  assert_lex3("%r{blah\\}blah}",
2292
- nil,
2293
- :tREGEXP_BEG, "%r{", EXPR_BEG, # FIX ?!?
2329
+ s(:lit, /blah\}blah/).line(1),
2330
+ :tREGEXP_BEG, "%r{", EXPR_BEG,
2294
2331
  :tSTRING_CONTENT, "blah\\}blah", EXPR_BEG,
2295
2332
  :tREGEXP_END, "", EXPR_LIT)
2296
2333
  end
2297
2334
 
2298
2335
  def test_yylex_regexp_escape_backslash_terminator_meta2
2299
- assert_lex3("%r/blah\\/blah/",
2300
- nil,
2301
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2302
- :tSTRING_CONTENT, "blah\\/blah", EXPR_BEG,
2303
- :tREGEXP_END, "", EXPR_LIT)
2336
+ rb = "%r/blah\\/blah/"
2337
+ pt = s(:lit, /blah\/blah/).line 1
2338
+
2339
+ assert_lex3(rb,
2340
+ pt,
2341
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2342
+ :tSTRING_CONTENT, "blah/blah", EXPR_BEG,
2343
+ :tREGEXP_END, "", EXPR_LIT)
2304
2344
  end
2305
2345
 
2306
2346
  def test_yylex_regexp_escape_backslash_terminator_meta3
2307
2347
  assert_lex3("%r/blah\\%blah/",
2308
2348
  nil,
2309
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2349
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2310
2350
  :tSTRING_CONTENT, "blah\\%blah", EXPR_BEG,
2311
2351
  :tREGEXP_END, "", EXPR_LIT)
2312
2352
  end
@@ -2316,8 +2356,9 @@ class TestRubyLexer < Minitest::Test
2316
2356
  end
2317
2357
 
2318
2358
  def test_yylex_regexp_escape_bs
2319
- assert_lex3("/regex\\\\regex/",
2320
- nil,
2359
+ rp = "/regex\\\\regex/"
2360
+ assert_lex3(rp,
2361
+ s(:lit, /regex\\regex/),
2321
2362
  :tREGEXP_BEG, "/", EXPR_BEG,
2322
2363
  :tSTRING_CONTENT, "regex\\\\regex", EXPR_BEG,
2323
2364
  :tREGEXP_END, "", EXPR_LIT)
@@ -2348,12 +2389,14 @@ class TestRubyLexer < Minitest::Test
2348
2389
  end
2349
2390
 
2350
2391
  def test_yylex_regexp_escape_double_backslash
2351
- regexp = '/[\\/\\\\]$/'
2352
- assert_lex3(regexp.dup,
2353
- nil,
2354
- :tREGEXP_BEG, "/", EXPR_BEG,
2355
- :tSTRING_CONTENT, "[\\/\\\\]$", EXPR_BEG,
2356
- :tREGEXP_END, "", EXPR_LIT)
2392
+ rb = '/[\\/\\\\]$/'
2393
+ pt = s(:lit, /[\/\\]$/)
2394
+
2395
+ assert_lex3(rb,
2396
+ pt,
2397
+ :tREGEXP_BEG, "/", EXPR_BEG,
2398
+ :tSTRING_CONTENT, "[/\\\\]$", EXPR_BEG,
2399
+ :tREGEXP_END, "", EXPR_LIT)
2357
2400
  end
2358
2401
 
2359
2402
  def test_yylex_regexp_escape_hex
@@ -2411,7 +2454,7 @@ class TestRubyLexer < Minitest::Test
2411
2454
  def test_yylex_regexp_escaped_delim
2412
2455
  assert_lex3("%r!blah(?\\!blah)!",
2413
2456
  nil,
2414
- :tREGEXP_BEG, "%r\000", EXPR_BEG,
2457
+ :tREGEXP_BEG, "%r\0", EXPR_BEG,
2415
2458
  :tSTRING_CONTENT, "blah(?!blah)", EXPR_BEG,
2416
2459
  :tREGEXP_END, "", EXPR_LIT)
2417
2460
  end
@@ -2526,7 +2569,9 @@ class TestRubyLexer < Minitest::Test
2526
2569
  end
2527
2570
 
2528
2571
  def test_yylex_string_bad_eos_quote
2529
- refute_lex("%{nest", :tSTRING_BEG, "%}")
2572
+ refute_lex("%{nest",
2573
+ :tSTRING_BEG, "%}",
2574
+ :tSTRING_CONTENT, "nest")
2530
2575
  end
2531
2576
 
2532
2577
  def test_yylex_string_double
@@ -2538,7 +2583,7 @@ class TestRubyLexer < Minitest::Test
2538
2583
  end
2539
2584
 
2540
2585
  def test_yylex_string_double_escape_C_backslash
2541
- assert_lex3("\"\\C-\\\\\"",
2586
+ assert_lex3(%W[ " \\ C - \\ \\ " ].join, # I hate escaping \ in ' and "
2542
2587
  nil,
2543
2588
  :tSTRING_BEG, "\"", EXPR_BEG,
2544
2589
  :tSTRING_CONTENT, "\034", EXPR_BEG,
@@ -2592,7 +2637,9 @@ class TestRubyLexer < Minitest::Test
2592
2637
  end
2593
2638
 
2594
2639
  def test_yylex_string_double_escape_c_backslash
2595
- refute_lex("\"\\c\\\"", :tSTRING_BEG, '"')
2640
+ refute_lex('"\\c\\"',
2641
+ :tSTRING_BEG, '"',
2642
+ :tSTRING_CONTENT, "\002")
2596
2643
  end
2597
2644
 
2598
2645
  def test_yylex_string_double_escape_c_escape
@@ -2628,12 +2675,12 @@ class TestRubyLexer < Minitest::Test
2628
2675
  nil,
2629
2676
  :tSTRING_BEG, "\"", EXPR_BEG,
2630
2677
  :tSTRING_CONTENT, "blah #x a ", EXPR_BEG,
2631
- :tSTRING_DVAR, nil, EXPR_BEG,
2678
+ :tSTRING_DVAR, "#", EXPR_BEG,
2632
2679
  :tSTRING_CONTENT, "@a b ", EXPR_BEG,
2633
- :tSTRING_DVAR, nil, EXPR_BEG,
2680
+ :tSTRING_DVAR, "#", EXPR_BEG,
2634
2681
  :tSTRING_CONTENT, "$b c ", EXPR_BEG,
2635
- :tSTRING_DBEG, nil, EXPR_BEG,
2636
- :tSTRING_CONTENT, "3} # ", EXPR_BEG,
2682
+ :tSTRING_DBEG, "#\{", EXPR_BEG,
2683
+ :tSTRING_CONTENT, "3} # ", EXPR_BEG, # FIX: wrong!?!?
2637
2684
  :tSTRING_END, "\"", EXPR_LIT)
2638
2685
  end
2639
2686
 
@@ -2667,12 +2714,12 @@ class TestRubyLexer < Minitest::Test
2667
2714
  nil,
2668
2715
  :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2669
2716
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2670
- :tSPACE, nil, EXPR_BEG,
2717
+ :tSPACE, " ", EXPR_BEG,
2671
2718
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2672
- :tSPACE, nil, EXPR_BEG,
2719
+ :tSPACE, " ", EXPR_BEG,
2673
2720
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2674
- :tSPACE, nil, EXPR_BEG,
2675
- :tSTRING_END, nil, EXPR_LIT)
2721
+ :tSPACE, "]", EXPR_BEG,
2722
+ :tSTRING_END, "]", EXPR_LIT)
2676
2723
  end
2677
2724
 
2678
2725
  def test_yylex_string_pct_I_extra_space
@@ -2680,12 +2727,12 @@ class TestRubyLexer < Minitest::Test
2680
2727
  nil,
2681
2728
  :tSYMBOLS_BEG, "%I[", EXPR_BEG,
2682
2729
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2683
- :tSPACE, nil, EXPR_BEG,
2730
+ :tSPACE, " ", EXPR_BEG,
2684
2731
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2685
- :tSPACE, nil, EXPR_BEG,
2732
+ :tSPACE, " ", EXPR_BEG,
2686
2733
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2687
- :tSPACE, nil, EXPR_BEG,
2688
- :tSTRING_END, nil, EXPR_LIT)
2734
+ :tSPACE, "]", EXPR_BEG,
2735
+ :tSTRING_END, "]", EXPR_LIT)
2689
2736
  end
2690
2737
 
2691
2738
  def test_yylex_string_pct_Q
@@ -2717,23 +2764,28 @@ class TestRubyLexer < Minitest::Test
2717
2764
  nil,
2718
2765
  :tWORDS_BEG, "%W[", EXPR_BEG,
2719
2766
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2720
- :tSPACE, nil, EXPR_BEG,
2767
+ :tSPACE, " ", EXPR_BEG,
2721
2768
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2722
- :tSPACE, nil, EXPR_BEG,
2769
+ :tSPACE, " ", EXPR_BEG,
2723
2770
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2724
- :tSPACE, nil, EXPR_BEG,
2725
- :tSTRING_END, nil, EXPR_LIT)
2771
+ :tSPACE, "]", EXPR_BEG,
2772
+ :tSTRING_END, "]", EXPR_LIT)
2726
2773
  end
2727
2774
 
2728
2775
  def test_yylex_string_pct_W_bs_nl
2729
- assert_lex3("%W[s1 \\\ns2]", # TODO: add interpolation to these
2730
- nil,
2776
+ rb = "%W[s1 \\\ns2]" # TODO: add interpolation to these
2777
+ pt = s(:array,
2778
+ s(:str, "s1").line(1),
2779
+ s(:str, "\ns2").line(1)).line(1)
2780
+
2781
+ assert_lex3(rb,
2782
+ pt,
2731
2783
  :tWORDS_BEG, "%W[", EXPR_BEG,
2732
2784
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2733
- :tSPACE, nil, EXPR_BEG,
2785
+ :tSPACE, " ", EXPR_BEG,
2734
2786
  :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2735
- :tSPACE, nil, EXPR_BEG,
2736
- :tSTRING_END, nil, EXPR_LIT)
2787
+ :tSPACE, "]", EXPR_BEG,
2788
+ :tSTRING_END, "]", EXPR_LIT)
2737
2789
  end
2738
2790
 
2739
2791
  def test_yylex_string_pct_angle
@@ -2749,12 +2801,12 @@ class TestRubyLexer < Minitest::Test
2749
2801
  nil,
2750
2802
  :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2751
2803
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2752
- :tSPACE, nil, EXPR_BEG,
2804
+ :tSPACE, " ", EXPR_BEG,
2753
2805
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2754
- :tSPACE, nil, EXPR_BEG,
2806
+ :tSPACE, " ", EXPR_BEG,
2755
2807
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2756
- :tSPACE, nil, EXPR_BEG,
2757
- :tSTRING_END, nil, EXPR_LIT)
2808
+ :tSPACE, "]", EXPR_BEG,
2809
+ :tSTRING_END, "]", EXPR_LIT)
2758
2810
  end
2759
2811
 
2760
2812
  def test_yylex_string_pct_i_extra_space
@@ -2762,12 +2814,12 @@ class TestRubyLexer < Minitest::Test
2762
2814
  nil,
2763
2815
  :tQSYMBOLS_BEG, "%i[", EXPR_BEG,
2764
2816
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2765
- :tSPACE, nil, EXPR_BEG,
2817
+ :tSPACE, " ", EXPR_BEG,
2766
2818
  :tSTRING_CONTENT, "s2", EXPR_BEG,
2767
- :tSPACE, nil, EXPR_BEG,
2819
+ :tSPACE, " ", EXPR_BEG,
2768
2820
  :tSTRING_CONTENT, "s3", EXPR_BEG,
2769
- :tSPACE, nil, EXPR_BEG,
2770
- :tSTRING_END, nil, EXPR_LIT)
2821
+ :tSPACE, "]", EXPR_BEG,
2822
+ :tSTRING_END, "]", EXPR_LIT)
2771
2823
  end
2772
2824
 
2773
2825
  def test_yylex_string_pct_other
@@ -2798,9 +2850,9 @@ class TestRubyLexer < Minitest::Test
2798
2850
  refute_lex("%w[s1 s2 ",
2799
2851
  :tQWORDS_BEG, "%w[",
2800
2852
  :tSTRING_CONTENT, "s1",
2801
- :tSPACE, nil,
2853
+ :tSPACE, " ",
2802
2854
  :tSTRING_CONTENT, "s2",
2803
- :tSPACE, nil)
2855
+ :tSPACE, " ")
2804
2856
  end
2805
2857
 
2806
2858
  def test_yylex_string_pct_w_bs_nl
@@ -2808,21 +2860,21 @@ class TestRubyLexer < Minitest::Test
2808
2860
  nil,
2809
2861
  :tQWORDS_BEG, "%w[", EXPR_BEG,
2810
2862
  :tSTRING_CONTENT, "s1", EXPR_BEG,
2811
- :tSPACE, nil, EXPR_BEG,
2863
+ :tSPACE, " ", EXPR_BEG,
2812
2864
  :tSTRING_CONTENT, "\ns2", EXPR_BEG,
2813
- :tSPACE, nil, EXPR_BEG,
2814
- :tSTRING_END, nil, EXPR_LIT)
2865
+ :tSPACE, "]", EXPR_BEG,
2866
+ :tSTRING_END, "]", EXPR_LIT)
2815
2867
  end
2816
2868
 
2817
2869
  def test_yylex_string_pct_w_bs_sp
2818
2870
  assert_lex3("%w[s\\ 1 s\\ 2]",
2819
- nil,
2871
+ s(:array, s(:str, "s 1"), s(:str, "s 2")),
2820
2872
  :tQWORDS_BEG, "%w[", EXPR_BEG,
2821
2873
  :tSTRING_CONTENT, "s 1", EXPR_BEG,
2822
- :tSPACE, nil, EXPR_BEG,
2874
+ :tSPACE, " ", EXPR_BEG,
2823
2875
  :tSTRING_CONTENT, "s 2", EXPR_BEG,
2824
- :tSPACE, nil, EXPR_BEG,
2825
- :tSTRING_END, nil, EXPR_LIT)
2876
+ :tSPACE, "]", EXPR_BEG,
2877
+ :tSTRING_END, "]", EXPR_LIT)
2826
2878
  end
2827
2879
 
2828
2880
  def test_yylex_string_single
@@ -2852,7 +2904,7 @@ class TestRubyLexer < Minitest::Test
2852
2904
  assert_lex3('"#@a\u{3024}"',
2853
2905
  s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, chr)),
2854
2906
  :tSTRING_BEG, '"', EXPR_BEG,
2855
- :tSTRING_DVAR, nil, EXPR_BEG,
2907
+ :tSTRING_DVAR, "#", EXPR_BEG,
2856
2908
  :tSTRING_CONTENT, "@a"+chr, EXPR_BEG,
2857
2909
  :tSTRING_END, '"', EXPR_LIT)
2858
2910
  end
@@ -2863,7 +2915,7 @@ class TestRubyLexer < Minitest::Test
2863
2915
 
2864
2916
  refute_lex('"#@a\u302zzz"',
2865
2917
  :tSTRING_BEG, '"',
2866
- :tSTRING_DVAR, nil,
2918
+ :tSTRING_DVAR, "#",
2867
2919
  :tSTRING_CONTENT, "@a"+str,
2868
2920
  :tSTRING_END, '"')
2869
2921
 
@@ -2872,7 +2924,7 @@ class TestRubyLexer < Minitest::Test
2872
2924
 
2873
2925
  refute_lex('"#@a\u30zzz"',
2874
2926
  :tSTRING_BEG, '"',
2875
- :tSTRING_DVAR, nil,
2927
+ :tSTRING_DVAR, "#",
2876
2928
  :tSTRING_CONTENT, "@a"+str,
2877
2929
  :tSTRING_END, '"')
2878
2930
 
@@ -2881,7 +2933,7 @@ class TestRubyLexer < Minitest::Test
2881
2933
 
2882
2934
  refute_lex('"#@a\u3zzz"',
2883
2935
  :tSTRING_BEG, '"',
2884
- :tSTRING_DVAR, nil,
2936
+ :tSTRING_DVAR, "#",
2885
2937
  :tSTRING_CONTENT, "@a"+str,
2886
2938
  :tSTRING_END, '"')
2887
2939
  end
@@ -2902,7 +2954,7 @@ class TestRubyLexer < Minitest::Test
2902
2954
  assert_lex3('"#@a\u3024abz"',
2903
2955
  s(:dstr, "", s(:evstr, s(:ivar, :@a)), s(:str, str)),
2904
2956
  :tSTRING_BEG, '"', EXPR_BEG,
2905
- :tSTRING_DVAR, nil, EXPR_BEG,
2957
+ :tSTRING_DVAR, "#", EXPR_BEG,
2906
2958
  :tSTRING_CONTENT, "@a"+str, EXPR_BEG,
2907
2959
  :tSTRING_END, '"', EXPR_LIT)
2908
2960
  end
@@ -2952,7 +3004,7 @@ class TestRubyLexer < Minitest::Test
2952
3004
  nil,
2953
3005
  :tSYMBEG, ":", EXPR_FNAME,
2954
3006
  :tSTRING_CONTENT, "symbol", EXPR_FNAME,
2955
- :tSTRING_DBEG, nil, EXPR_FNAME,
3007
+ :tSTRING_DBEG, '#{', EXPR_FNAME,
2956
3008
  :tSTRING_CONTENT, "1+1}", EXPR_FNAME, # HUH? this is BS
2957
3009
  :tSTRING_END, "\"", EXPR_LIT)
2958
3010
  end