parser 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Rakefile +2 -1
- data/TODO.md +20 -0
- data/bin/explain-parse +1 -1
- data/lib/parser/all.rb +1 -0
- data/lib/parser/builders/default.rb +30 -2
- data/lib/parser/lexer/literal.rb +41 -20
- data/lib/parser/lexer/stack_state.rb +4 -0
- data/lib/parser/lexer.rb +2585 -2545
- data/lib/parser/lexer.rl +61 -38
- data/lib/parser/ruby18.rb +3133 -3171
- data/lib/parser/ruby18.y +16 -30
- data/lib/parser/ruby19.rb +3319 -3388
- data/lib/parser/ruby19.y +35 -50
- data/lib/parser/ruby20.rb +6961 -0
- data/lib/parser/ruby20.y +2190 -0
- data/lib/parser.rb +0 -1
- data/parse.y.diff +12201 -0
- data/parser.gemspec +2 -1
- data/test/helper.rb +1 -1
- data/test/parse_helper.rb +2 -2
- data/test/test_lexer.rb +95 -5
- data/test/test_lexer_stack_state.rb +7 -0
- data/test/test_parser.rb +138 -5
- metadata +6 -2
data/lib/parser/lexer.rl
CHANGED
@@ -265,14 +265,18 @@ class Parser::Lexer
|
|
265
265
|
new_literal = Literal.new(self, *args)
|
266
266
|
@literal_stack.push(new_literal)
|
267
267
|
|
268
|
-
if new_literal.
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
268
|
+
if new_literal.words?
|
269
|
+
if new_literal.interpolate?
|
270
|
+
self.class.lex_en_interp_words
|
271
|
+
else
|
272
|
+
self.class.lex_en_plain_words
|
273
|
+
end
|
274
274
|
else
|
275
|
-
|
275
|
+
if new_literal.interpolate?
|
276
|
+
self.class.lex_en_interp_string
|
277
|
+
else
|
278
|
+
self.class.lex_en_plain_string
|
279
|
+
end
|
276
280
|
end
|
277
281
|
end
|
278
282
|
|
@@ -313,9 +317,9 @@ class Parser::Lexer
|
|
313
317
|
}
|
314
318
|
|
315
319
|
PUNCTUATION_BEGIN = {
|
316
|
-
'&' => :tAMPER, '*' => :tSTAR,
|
317
|
-
'-' => :tUMINUS, '::' => :tCOLON3,
|
318
|
-
'{' => :tLBRACE, '[' => :tLBRACK,
|
320
|
+
'&' => :tAMPER, '*' => :tSTAR, '**' => :tDSTAR,
|
321
|
+
'+' => :tUPLUS, '-' => :tUMINUS, '::' => :tCOLON3,
|
322
|
+
'(' => :tLPAREN, '{' => :tLBRACE, '[' => :tLBRACK,
|
319
323
|
}
|
320
324
|
|
321
325
|
KEYWORDS = {
|
@@ -410,7 +414,8 @@ class Parser::Lexer
|
|
410
414
|
|
411
415
|
# A list of punctuation which has different meaning when used at the
|
412
416
|
# beginning of expression.
|
413
|
-
punctuation_begin = '-' | '+' | '::' | '(' | '[' |
|
417
|
+
punctuation_begin = '-' | '+' | '::' | '(' | '[' |
|
418
|
+
'*' | '**' | '&' ;
|
414
419
|
|
415
420
|
# A list of all punctuation except punctuation_begin.
|
416
421
|
punctuation_end = ',' | '=' | '->' | '(' | '[' | ']' |
|
@@ -748,6 +753,10 @@ class Parser::Lexer
|
|
748
753
|
literal.flush_string
|
749
754
|
}
|
750
755
|
|
756
|
+
action extend_string_space {
|
757
|
+
literal.extend_space @ts, @te
|
758
|
+
}
|
759
|
+
|
751
760
|
#
|
752
761
|
# === INTERPOLATION PARSING ===
|
753
762
|
#
|
@@ -759,6 +768,8 @@ class Parser::Lexer
|
|
759
768
|
|
760
769
|
action extend_interp_var {
|
761
770
|
literal.flush_string
|
771
|
+
literal.extend_content
|
772
|
+
|
762
773
|
emit(:tSTRING_DVAR, nil, @ts, @ts + 1)
|
763
774
|
|
764
775
|
p = @ts
|
@@ -790,10 +801,10 @@ class Parser::Lexer
|
|
790
801
|
e_rbrace = '}' % {
|
791
802
|
if literal
|
792
803
|
if literal.end_interp_brace_and_try_closing
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
emit(:
|
804
|
+
if version?(18, 19)
|
805
|
+
emit(:tRCURLY, '}')
|
806
|
+
else
|
807
|
+
emit(:tSTRING_DEND, '}')
|
797
808
|
end
|
798
809
|
|
799
810
|
if literal.saved_herebody_s
|
@@ -809,6 +820,8 @@ class Parser::Lexer
|
|
809
820
|
|
810
821
|
action extend_interp_code {
|
811
822
|
literal.flush_string
|
823
|
+
literal.extend_content
|
824
|
+
|
812
825
|
emit(:tSTRING_DBEG, '#{')
|
813
826
|
|
814
827
|
literal.saved_herebody_s = @herebody_s
|
@@ -825,7 +838,7 @@ class Parser::Lexer
|
|
825
838
|
interp_code => extend_interp_code;
|
826
839
|
interp_var => extend_interp_var;
|
827
840
|
e_bs escape => extend_string_escaped;
|
828
|
-
c_space_nl
|
841
|
+
c_space_nl+ => extend_string_space;
|
829
842
|
c_eol => extend_string_eol;
|
830
843
|
c_any => extend_string;
|
831
844
|
*|;
|
@@ -840,7 +853,7 @@ class Parser::Lexer
|
|
840
853
|
|
841
854
|
plain_words := |*
|
842
855
|
e_bs c_any => extend_string_escaped;
|
843
|
-
c_space_nl
|
856
|
+
c_space_nl+ => extend_string_space;
|
844
857
|
c_eol => extend_string_eol;
|
845
858
|
c_any => extend_string;
|
846
859
|
*|;
|
@@ -999,7 +1012,9 @@ class Parser::Lexer
|
|
999
1012
|
=> { p = @ts - 1
|
1000
1013
|
fcall expr_variable; };
|
1001
1014
|
|
1002
|
-
|
1015
|
+
# TODO whitespace rule
|
1016
|
+
c_space;
|
1017
|
+
e_heredoc_nl;
|
1003
1018
|
|
1004
1019
|
c_any
|
1005
1020
|
=> { fhold; fgoto expr_end; };
|
@@ -1030,9 +1045,11 @@ class Parser::Lexer
|
|
1030
1045
|
=> { emit_table(PUNCTUATION)
|
1031
1046
|
fnext expr_arg; fbreak; };
|
1032
1047
|
|
1033
|
-
|
1034
|
-
|
1035
|
-
|
1048
|
+
# TODO whitespace rule
|
1049
|
+
c_space;
|
1050
|
+
e_heredoc_nl;
|
1051
|
+
'\\' e_heredoc_nl;
|
1052
|
+
'#' c_line*;
|
1036
1053
|
|
1037
1054
|
c_any
|
1038
1055
|
=> { fhold; fgoto expr_end; };
|
@@ -1108,12 +1125,12 @@ class Parser::Lexer
|
|
1108
1125
|
};
|
1109
1126
|
|
1110
1127
|
# x *1
|
1111
|
-
# Ambiguous splat or block-pass.
|
1112
|
-
c_space+
|
1128
|
+
# Ambiguous splat, kwsplat or block-pass.
|
1129
|
+
c_space+ %{ tm = p } ( '*' | '&' | '**' )
|
1113
1130
|
=> {
|
1114
|
-
message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(
|
1131
|
+
message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(tm, @te) }
|
1115
1132
|
diagnostic :warning, message,
|
1116
|
-
range(
|
1133
|
+
range(tm, @te)
|
1117
1134
|
|
1118
1135
|
fhold; fgoto expr_beg;
|
1119
1136
|
};
|
@@ -1182,7 +1199,7 @@ class Parser::Lexer
|
|
1182
1199
|
=> { emit(:kDO_BLOCK)
|
1183
1200
|
fnext expr_value; };
|
1184
1201
|
|
1185
|
-
c_space
|
1202
|
+
c_space;
|
1186
1203
|
|
1187
1204
|
c_any
|
1188
1205
|
=> { fhold; fgoto expr_end; };
|
@@ -1200,11 +1217,11 @@ class Parser::Lexer
|
|
1200
1217
|
=> { emit_table(KEYWORDS)
|
1201
1218
|
fnext expr_beg; fbreak; };
|
1202
1219
|
|
1203
|
-
|
1204
|
-
|
1220
|
+
# TODO whitespace rule
|
1221
|
+
c_space;
|
1205
1222
|
'#' c_line*;
|
1206
1223
|
|
1207
|
-
|
1224
|
+
e_heredoc_nl
|
1208
1225
|
=> { fhold; fgoto expr_end; };
|
1209
1226
|
|
1210
1227
|
c_any
|
@@ -1430,7 +1447,8 @@ class Parser::Lexer
|
|
1430
1447
|
=> {
|
1431
1448
|
emit(:tIDENTIFIER)
|
1432
1449
|
|
1433
|
-
if
|
1450
|
+
if !version?(18) &&
|
1451
|
+
!@static_env.nil? && @static_env.declared?(tok)
|
1434
1452
|
fnext expr_end; fbreak;
|
1435
1453
|
else
|
1436
1454
|
fnext expr_arg; fbreak;
|
@@ -1441,15 +1459,16 @@ class Parser::Lexer
|
|
1441
1459
|
# WHITESPACE
|
1442
1460
|
#
|
1443
1461
|
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1462
|
+
# TODO whitespace rule
|
1463
|
+
c_space;
|
1464
|
+
e_heredoc_nl;
|
1465
|
+
'\\' e_heredoc_nl;
|
1447
1466
|
|
1448
1467
|
'#' c_line* c_eol
|
1449
1468
|
=> { @comments << tok
|
1450
1469
|
fhold; };
|
1451
1470
|
|
1452
|
-
|
1471
|
+
e_heredoc_nl '=begin' ( c_space | c_eol )
|
1453
1472
|
=> { p = @ts - 1
|
1454
1473
|
fgoto line_begin; };
|
1455
1474
|
|
@@ -1476,7 +1495,9 @@ class Parser::Lexer
|
|
1476
1495
|
=> { p = @ts - 1
|
1477
1496
|
fgoto expr_end; };
|
1478
1497
|
|
1479
|
-
|
1498
|
+
# TODO whitespace rule
|
1499
|
+
c_space;
|
1500
|
+
e_heredoc_nl;
|
1480
1501
|
|
1481
1502
|
c_any
|
1482
1503
|
=> { fhold; fgoto expr_beg; };
|
@@ -1489,9 +1510,9 @@ class Parser::Lexer
|
|
1489
1510
|
# STABBY LAMBDA
|
1490
1511
|
#
|
1491
1512
|
|
1492
|
-
'->'
|
1513
|
+
'->' c_space*
|
1493
1514
|
=> {
|
1494
|
-
emit_table(PUNCTUATION)
|
1515
|
+
emit_table(PUNCTUATION, @ts, @ts + 2)
|
1495
1516
|
|
1496
1517
|
@lambda_stack.push @paren_nest
|
1497
1518
|
fbreak;
|
@@ -1535,7 +1556,7 @@ class Parser::Lexer
|
|
1535
1556
|
'class' c_space_nl* '<<'
|
1536
1557
|
=> { emit(:kCLASS, 'class', @ts, @ts + 5)
|
1537
1558
|
emit(:tLSHFT, '<<', @te - 2, @te)
|
1538
|
-
fnext
|
1559
|
+
fnext expr_value; fbreak; };
|
1539
1560
|
|
1540
1561
|
# a if b:c: Syntax error.
|
1541
1562
|
keyword_modifier
|
@@ -1717,6 +1738,7 @@ class Parser::Lexer
|
|
1717
1738
|
# WHITESPACE
|
1718
1739
|
#
|
1719
1740
|
|
1741
|
+
# TODO whitespace rule
|
1720
1742
|
'\\' e_heredoc_nl;
|
1721
1743
|
|
1722
1744
|
'\\' c_line {
|
@@ -1779,6 +1801,7 @@ class Parser::Lexer
|
|
1779
1801
|
*|;
|
1780
1802
|
|
1781
1803
|
line_begin := |*
|
1804
|
+
# TODO whitespace rule
|
1782
1805
|
c_space_nl+;
|
1783
1806
|
|
1784
1807
|
'#' c_line* c_eol
|