ruby_parser 3.20.0 → 3.20.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +1 -1
- data/History.rdoc +24 -0
- data/compare/normalize.rb +1 -0
- data/lib/ruby20_parser.rb +4266 -4283
- data/lib/ruby20_parser.y +50 -26
- data/lib/ruby21_parser.rb +4240 -4239
- data/lib/ruby21_parser.y +50 -26
- data/lib/ruby22_parser.rb +4288 -4289
- data/lib/ruby22_parser.y +50 -26
- data/lib/ruby23_parser.rb +4273 -4242
- data/lib/ruby23_parser.y +50 -26
- data/lib/ruby24_parser.rb +4278 -4297
- data/lib/ruby24_parser.y +50 -26
- data/lib/ruby25_parser.rb +4269 -4288
- data/lib/ruby25_parser.y +50 -26
- data/lib/ruby26_parser.rb +4269 -4288
- data/lib/ruby26_parser.y +50 -26
- data/lib/ruby27_parser.rb +4172 -4205
- data/lib/ruby27_parser.y +50 -26
- data/lib/ruby30_parser.rb +6028 -5970
- data/lib/ruby30_parser.y +135 -86
- data/lib/ruby31_parser.rb +6194 -6183
- data/lib/ruby31_parser.y +136 -87
- data/lib/ruby32_parser.rb +6290 -6327
- data/lib/ruby32_parser.y +157 -107
- data/lib/ruby3_parser.yy +172 -87
- data/lib/ruby_lexer.rb +15 -7
- data/lib/ruby_lexer.rex.rb +1 -1
- data/lib/ruby_lexer_strings.rb +2 -2
- data/lib/ruby_parser.yy +50 -26
- data/lib/ruby_parser_extras.rb +25 -19
- data/test/test_ruby_parser.rb +195 -7
- data/tools/munge.rb +8 -2
- data/tools/ripper.rb +14 -12
- data.tar.gz.sig +0 -0
- metadata +3 -3
- metadata.gz.sig +0 -0
data/lib/ruby_lexer.rb
CHANGED
@@ -117,7 +117,7 @@ class RubyLexer
|
|
117
117
|
|
118
118
|
def initialize _ = nil
|
119
119
|
@lex_state = nil # remove one warning under $DEBUG
|
120
|
-
|
120
|
+
@lex_state = EXPR_NONE
|
121
121
|
|
122
122
|
self.cond = RubyParserStuff::StackState.new(:cond, $DEBUG)
|
123
123
|
self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
|
@@ -271,6 +271,8 @@ class RubyLexer
|
|
271
271
|
nil # TODO
|
272
272
|
end
|
273
273
|
|
274
|
+
# TODO: make all tXXXX terminals include lexer.lineno ... enforce it somehow?
|
275
|
+
|
274
276
|
def process_brace_close text
|
275
277
|
case matched
|
276
278
|
when "}" then
|
@@ -353,8 +355,14 @@ class RubyLexer
|
|
353
355
|
end
|
354
356
|
end
|
355
357
|
|
356
|
-
def process_dots text
|
357
|
-
|
358
|
+
def process_dots text # parse32.y:10216
|
359
|
+
is_beg = self.is_beg?
|
360
|
+
self.lex_state = EXPR_BEG
|
361
|
+
|
362
|
+
return result EXPR_ENDARG, :tBDOT3, text if
|
363
|
+
parser.in_argdef && text == "..." # TODO: version check?
|
364
|
+
|
365
|
+
tokens = ruby27plus? && is_beg ? BTOKENS : TOKENS
|
358
366
|
|
359
367
|
result EXPR_BEG, tokens[text], text
|
360
368
|
end
|
@@ -687,7 +695,7 @@ class RubyLexer
|
|
687
695
|
return process_token_keyword keyword if keyword
|
688
696
|
end
|
689
697
|
|
690
|
-
# matching: compare/
|
698
|
+
# matching: compare/parse32.y:9031
|
691
699
|
state = if lex_state =~ EXPR_BEG_ANY|EXPR_ARG_ANY|EXPR_DOT then
|
692
700
|
cmd_state ? EXPR_CMDARG : EXPR_ARG
|
693
701
|
elsif lex_state =~ EXPR_FNAME then
|
@@ -709,7 +717,7 @@ class RubyLexer
|
|
709
717
|
end
|
710
718
|
|
711
719
|
def process_token_keyword keyword
|
712
|
-
# matching MIDDLE of parse_ident in compare/
|
720
|
+
# matching MIDDLE of parse_ident in compare/parse32.y:9695
|
713
721
|
state = lex_state
|
714
722
|
|
715
723
|
return result(EXPR_ENDFN, keyword.id0, token) if lex_state =~ EXPR_FNAME
|
@@ -718,7 +726,7 @@ class RubyLexer
|
|
718
726
|
self.command_start = true if lex_state =~ EXPR_BEG
|
719
727
|
|
720
728
|
case
|
721
|
-
when keyword.id0 == :kDO then #
|
729
|
+
when keyword.id0 == :kDO then # parse32.y line 9712
|
722
730
|
case
|
723
731
|
when lambda_beginning? then
|
724
732
|
self.lpar_beg = nil # lambda_beginning? == FALSE in the body of "-> do ... end"
|
@@ -890,7 +898,7 @@ class RubyLexer
|
|
890
898
|
end
|
891
899
|
|
892
900
|
def inspect
|
893
|
-
return "
|
901
|
+
return "EXPR_NONE" if n.zero? # HACK?
|
894
902
|
|
895
903
|
names.map { |v, k| k if self =~ v }.
|
896
904
|
compact.
|
data/lib/ruby_lexer.rex.rb
CHANGED
data/lib/ruby_lexer_strings.rb
CHANGED
@@ -56,7 +56,7 @@ class RubyLexer
|
|
56
56
|
return :tSTRING_DVAR, matched
|
57
57
|
when scan(/#[{]/) then
|
58
58
|
self.command_start = true
|
59
|
-
return :tSTRING_DBEG, matched
|
59
|
+
return :tSTRING_DBEG, [matched, lineno]
|
60
60
|
when scan(/#/) then
|
61
61
|
string_buffer << "#"
|
62
62
|
end
|
@@ -415,7 +415,7 @@ class RubyLexer
|
|
415
415
|
return :tSTRING_DVAR, matched
|
416
416
|
when scan(/#[{]/) then
|
417
417
|
self.command_start = true
|
418
|
-
return :tSTRING_DBEG, matched
|
418
|
+
return :tSTRING_DBEG, [matched, lineno]
|
419
419
|
when scan(/#/) then
|
420
420
|
# do nothing but swallow
|
421
421
|
end
|
data/lib/ruby_parser.yy
CHANGED
@@ -253,6 +253,7 @@ rule
|
|
253
253
|
| lhs tEQL mrhs
|
254
254
|
{
|
255
255
|
lhs, _, rhs = val
|
256
|
+
|
256
257
|
result = new_assign lhs, s(:svalue, rhs).line(rhs.line)
|
257
258
|
}
|
258
259
|
#if V == 20
|
@@ -1107,8 +1108,10 @@ rule
|
|
1107
1108
|
|
1108
1109
|
paren_args: tLPAREN2 opt_call_args rparen
|
1109
1110
|
{
|
1110
|
-
_, args, _ = val
|
1111
|
+
_, args, (_, line_max) = val
|
1112
|
+
|
1111
1113
|
result = args
|
1114
|
+
result.line_max = line_max if args
|
1112
1115
|
}
|
1113
1116
|
#if V >= 27
|
1114
1117
|
| tLPAREN2 args tCOMMA args_forward rparen
|
@@ -1290,7 +1293,6 @@ rule
|
|
1290
1293
|
| k_begin
|
1291
1294
|
{
|
1292
1295
|
lexer.cmdarg.push false
|
1293
|
-
result = self.lexer.lineno
|
1294
1296
|
}
|
1295
1297
|
bodystmt k_end
|
1296
1298
|
{
|
@@ -1335,12 +1337,14 @@ rule
|
|
1335
1337
|
{
|
1336
1338
|
result = wrap :colon3, val[1]
|
1337
1339
|
}
|
1338
|
-
| tLBRACK { result = lexer.lineno } aref_args
|
1340
|
+
| tLBRACK { result = lexer.lineno } aref_args rbracket
|
1339
1341
|
{
|
1340
|
-
_, line, args, _ = val
|
1342
|
+
_, line, args, (_, line_max) = val
|
1343
|
+
|
1341
1344
|
result = args || s(:array)
|
1342
1345
|
result.sexp_type = :array # aref_args is :args
|
1343
1346
|
result.line line
|
1347
|
+
result.line_max = line_max
|
1344
1348
|
}
|
1345
1349
|
| tLBRACE
|
1346
1350
|
{
|
@@ -1453,9 +1457,6 @@ rule
|
|
1453
1457
|
result = new_for iter, var, body
|
1454
1458
|
}
|
1455
1459
|
| k_class
|
1456
|
-
{
|
1457
|
-
result = self.lexer.lineno
|
1458
|
-
}
|
1459
1460
|
cpath superclass
|
1460
1461
|
{
|
1461
1462
|
if (self.in_def || self.in_single > 0) then
|
@@ -1470,9 +1471,6 @@ rule
|
|
1470
1471
|
self.lexer.ignore_body_comments
|
1471
1472
|
}
|
1472
1473
|
| k_class tLSHFT
|
1473
|
-
{
|
1474
|
-
result = self.lexer.lineno
|
1475
|
-
}
|
1476
1474
|
expr
|
1477
1475
|
{
|
1478
1476
|
result = self.in_def
|
@@ -1491,9 +1489,6 @@ rule
|
|
1491
1489
|
self.lexer.ignore_body_comments
|
1492
1490
|
}
|
1493
1491
|
| k_module
|
1494
|
-
{
|
1495
|
-
result = self.lexer.lineno
|
1496
|
-
}
|
1497
1492
|
cpath
|
1498
1493
|
{
|
1499
1494
|
yyerror "module definition in method body" if
|
@@ -1942,13 +1937,19 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
1942
1937
|
{
|
1943
1938
|
call, args = val
|
1944
1939
|
|
1945
|
-
result = call
|
1940
|
+
result = call
|
1941
|
+
|
1942
|
+
if args then
|
1943
|
+
call.concat args.sexp_body
|
1944
|
+
result.line_max = args.line_max
|
1945
|
+
end
|
1946
1946
|
}
|
1947
1947
|
| primary_value call_op operation2 opt_paren_args
|
1948
1948
|
{
|
1949
|
-
recv, call_op, (op,
|
1949
|
+
recv, call_op, (op, op_line), args = val
|
1950
1950
|
|
1951
1951
|
result = new_call recv, op.to_sym, args, call_op
|
1952
|
+
result.line_max = op_line unless args
|
1952
1953
|
}
|
1953
1954
|
| primary_value tCOLON2 operation2 paren_args
|
1954
1955
|
{
|
@@ -2674,15 +2675,17 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2674
2675
|
|
2675
2676
|
words: tWORDS_BEG tSPACE tSTRING_END
|
2676
2677
|
{
|
2677
|
-
(_, line), _, _ = val
|
2678
|
+
(_, line), _, (_, line_max) = val
|
2678
2679
|
|
2679
2680
|
result = s(:array).line line
|
2681
|
+
result.line_max = line_max
|
2680
2682
|
}
|
2681
2683
|
| tWORDS_BEG word_list tSTRING_END
|
2682
2684
|
{
|
2683
|
-
(_, line), list, _ = val
|
2685
|
+
(_, line), list, (_, line_max) = val
|
2684
2686
|
|
2685
2687
|
result = list.line line
|
2688
|
+
result.line_max = line_max
|
2686
2689
|
}
|
2687
2690
|
|
2688
2691
|
word_list: none
|
@@ -2702,15 +2705,17 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2702
2705
|
|
2703
2706
|
symbols: tSYMBOLS_BEG tSPACE tSTRING_END
|
2704
2707
|
{
|
2705
|
-
(_, line), _, _ = val
|
2708
|
+
(_, line), _, (_, line_max) = val
|
2706
2709
|
|
2707
2710
|
result = s(:array).line line
|
2711
|
+
result.line_max = line_max
|
2708
2712
|
}
|
2709
2713
|
| tSYMBOLS_BEG symbol_list tSTRING_END
|
2710
2714
|
{
|
2711
|
-
(_, line), list, _, = val
|
2712
|
-
|
2713
|
-
result = list
|
2715
|
+
(_, line), list, (_, line_max), = val
|
2716
|
+
|
2717
|
+
result = list.line line
|
2718
|
+
result.line_max = line_max
|
2714
2719
|
}
|
2715
2720
|
|
2716
2721
|
symbol_list: none
|
@@ -2725,28 +2730,32 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2725
2730
|
|
2726
2731
|
qwords: tQWORDS_BEG tSPACE tSTRING_END
|
2727
2732
|
{
|
2728
|
-
(_, line), _, _ = val
|
2733
|
+
(_, line), _, (_, line_max) = val
|
2729
2734
|
|
2730
2735
|
result = s(:array).line line
|
2736
|
+
result.line_max = line_max
|
2731
2737
|
}
|
2732
2738
|
| tQWORDS_BEG qword_list tSTRING_END
|
2733
2739
|
{
|
2734
|
-
(_, line), list, _ = val
|
2740
|
+
(_, line), list, (_, line_max) = val
|
2735
2741
|
|
2736
2742
|
result = list.line line
|
2743
|
+
result.line_max = line_max
|
2737
2744
|
}
|
2738
2745
|
|
2739
2746
|
qsymbols: tQSYMBOLS_BEG tSPACE tSTRING_END
|
2740
2747
|
{
|
2741
|
-
(_, line), _, _ = val
|
2748
|
+
(_, line), _, (_, line_max) = val
|
2742
2749
|
|
2743
2750
|
result = s(:array).line line
|
2751
|
+
result.line_max = line_max
|
2744
2752
|
}
|
2745
2753
|
| tQSYMBOLS_BEG qsym_list tSTRING_END
|
2746
2754
|
{
|
2747
|
-
(_, line), list, _ = val
|
2755
|
+
(_, line), list, (_, line_max) = val
|
2748
2756
|
|
2749
2757
|
result = list.line line
|
2758
|
+
result.line_max = line_max
|
2750
2759
|
}
|
2751
2760
|
|
2752
2761
|
qword_list: none
|
@@ -3197,7 +3206,14 @@ keyword_variable: kNIL { result = s(:nil).line lexer.lineno }
|
|
3197
3206
|
result = s(:args, list).line list.line
|
3198
3207
|
end
|
3199
3208
|
|
3200
|
-
|
3209
|
+
if Sexp === item then
|
3210
|
+
line_max = item.line_max
|
3211
|
+
else
|
3212
|
+
item, line_max = item
|
3213
|
+
end
|
3214
|
+
|
3215
|
+
result << item
|
3216
|
+
result.line_max = line_max
|
3201
3217
|
}
|
3202
3218
|
|
3203
3219
|
#if V == 20
|
@@ -3449,7 +3465,15 @@ keyword_variable: kNIL { result = s(:nil).line lexer.lineno }
|
|
3449
3465
|
opt_terms: | terms
|
3450
3466
|
opt_nl: | tNL
|
3451
3467
|
rparen: opt_nl tRPAREN
|
3468
|
+
{
|
3469
|
+
_, close = val
|
3470
|
+
result = [close, lexer.lineno]
|
3471
|
+
}
|
3452
3472
|
rbracket: opt_nl tRBRACK
|
3473
|
+
{
|
3474
|
+
_, close = val
|
3475
|
+
result = [close, lexer.lineno]
|
3476
|
+
}
|
3453
3477
|
#if V >= 27
|
3454
3478
|
rbrace: opt_nl tRCURLY
|
3455
3479
|
#endif
|
data/lib/ruby_parser_extras.rb
CHANGED
@@ -18,7 +18,7 @@ class Sexp
|
|
18
18
|
end
|
19
19
|
|
20
20
|
##
|
21
|
-
# Returns the
|
21
|
+
# Returns the minimum line number of the children of self.
|
22
22
|
|
23
23
|
def line_min
|
24
24
|
@line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
|
@@ -30,9 +30,9 @@ class Sexp
|
|
30
30
|
end
|
31
31
|
|
32
32
|
module RubyParserStuff
|
33
|
-
VERSION = "3.20.
|
33
|
+
VERSION = "3.20.3"
|
34
34
|
|
35
|
-
attr_accessor :lexer, :in_def, :in_single, :file
|
35
|
+
attr_accessor :lexer, :in_def, :in_single, :file, :in_argdef
|
36
36
|
attr_accessor :in_kwarg
|
37
37
|
attr_reader :env, :comments
|
38
38
|
|
@@ -122,6 +122,7 @@ module RubyParserStuff
|
|
122
122
|
self.lexer = RubyLexer.new v && v.to_i
|
123
123
|
self.lexer.parser = self
|
124
124
|
self.in_kwarg = false
|
125
|
+
self.in_argdef = false
|
125
126
|
|
126
127
|
@env = RubyParserStuff::Environment.new
|
127
128
|
@comments = []
|
@@ -153,6 +154,7 @@ module RubyParserStuff
|
|
153
154
|
result.line lexer.lineno
|
154
155
|
else
|
155
156
|
result.line ss.first.line
|
157
|
+
result.line_max = ss.first.line_max
|
156
158
|
end
|
157
159
|
|
158
160
|
args.each do |arg|
|
@@ -330,9 +332,8 @@ module RubyParserStuff
|
|
330
332
|
end
|
331
333
|
|
332
334
|
args.each do |arg|
|
333
|
-
|
334
|
-
|
335
|
-
end
|
335
|
+
# ruby 3.0+ TODO: next if arg in [String, Integer] # eg ["(", 1]
|
336
|
+
next if arg.class == Array && arg.map(&:class) == [String, Integer]
|
336
337
|
|
337
338
|
case arg
|
338
339
|
when Sexp then
|
@@ -794,6 +795,7 @@ module RubyParserStuff
|
|
794
795
|
case lhs.sexp_type
|
795
796
|
when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
|
796
797
|
lhs << rhs
|
798
|
+
lhs.line_max = rhs.line_max
|
797
799
|
when :const then
|
798
800
|
lhs.sexp_type = :cdecl
|
799
801
|
lhs << rhs
|
@@ -823,10 +825,10 @@ module RubyParserStuff
|
|
823
825
|
end
|
824
826
|
|
825
827
|
def new_begin val
|
826
|
-
_,
|
828
|
+
(_, line), _, body, _ = val
|
827
829
|
|
828
830
|
result = body ? s(:begin, body) : s(:nil)
|
829
|
-
result.line
|
831
|
+
result.line line
|
830
832
|
|
831
833
|
result
|
832
834
|
end
|
@@ -885,12 +887,13 @@ module RubyParserStuff
|
|
885
887
|
# TODO: need a test with f(&b) to produce block_pass
|
886
888
|
# TODO: need a test with f(&b) { } to produce warning
|
887
889
|
|
888
|
-
if args
|
890
|
+
if args then
|
889
891
|
if ARG_TYPES[args.sexp_type] then
|
890
892
|
result.concat args.sexp_body
|
891
893
|
else
|
892
894
|
result << args
|
893
895
|
end
|
896
|
+
result.line_max = args.line_max
|
894
897
|
end
|
895
898
|
|
896
899
|
# line = result.grep(Sexp).map(&:line).compact.min
|
@@ -926,8 +929,7 @@ module RubyParserStuff
|
|
926
929
|
end
|
927
930
|
|
928
931
|
def new_class val
|
929
|
-
|
930
|
-
line, path, superclass, body = val[1], val[2], val[3], val[5]
|
932
|
+
(_, line), path, superclass, _, body, (_, line_max) = val
|
931
933
|
|
932
934
|
path = path.first if path.instance_of? Array
|
933
935
|
|
@@ -942,6 +944,7 @@ module RubyParserStuff
|
|
942
944
|
end
|
943
945
|
|
944
946
|
result.line = line
|
947
|
+
result.line_max = line_max
|
945
948
|
result.comments = self.comments.pop
|
946
949
|
result
|
947
950
|
end
|
@@ -970,13 +973,14 @@ module RubyParserStuff
|
|
970
973
|
end
|
971
974
|
|
972
975
|
def new_defn val
|
973
|
-
_, (name, line), in_def, args, body, _ = val
|
976
|
+
_, (name, line), in_def, args, body, (_, line_max) = val
|
974
977
|
|
975
978
|
body ||= s(:nil).line line
|
976
979
|
|
977
980
|
args.line line
|
978
981
|
|
979
982
|
result = s(:defn, name.to_sym, args).line line
|
983
|
+
result.line_max = line_max
|
980
984
|
|
981
985
|
if body.sexp_type == :block then
|
982
986
|
result.push(*body.sexp_body)
|
@@ -1033,13 +1037,14 @@ module RubyParserStuff
|
|
1033
1037
|
end
|
1034
1038
|
|
1035
1039
|
def new_defs val
|
1036
|
-
_, recv, (name, line), in_def, args, body, _ = val
|
1040
|
+
_, recv, (name, line), in_def, args, body, (_, line_max) = val
|
1037
1041
|
|
1038
1042
|
body ||= s(:nil).line line
|
1039
1043
|
|
1040
1044
|
args.line line
|
1041
1045
|
|
1042
1046
|
result = s(:defs, recv, name.to_sym, args).line line
|
1047
|
+
result.line_max = line_max
|
1043
1048
|
|
1044
1049
|
# TODO: remove_begin
|
1045
1050
|
# TODO: reduce_nodes
|
@@ -1204,12 +1209,12 @@ module RubyParserStuff
|
|
1204
1209
|
end
|
1205
1210
|
|
1206
1211
|
def new_module val
|
1207
|
-
|
1208
|
-
line, path, body = val[1], val[2], val[4]
|
1212
|
+
(_, line_min), path, _, body, (_, line_max) = val
|
1209
1213
|
|
1210
1214
|
path = path.first if path.instance_of? Array
|
1211
1215
|
|
1212
|
-
result = s(:module, path).line
|
1216
|
+
result = s(:module, path).line line_min
|
1217
|
+
result.line_max = line_max
|
1213
1218
|
|
1214
1219
|
if body then # REFACTOR?
|
1215
1220
|
if body.sexp_type == :block then
|
@@ -1291,9 +1296,10 @@ module RubyParserStuff
|
|
1291
1296
|
end
|
1292
1297
|
|
1293
1298
|
def new_regexp val
|
1294
|
-
(_, line), node, (options,
|
1299
|
+
(_, line), node, (options, line_max) = val
|
1295
1300
|
|
1296
1301
|
node ||= s(:str, "").line line
|
1302
|
+
node.line_max = line_max
|
1297
1303
|
|
1298
1304
|
o, k = 0, nil
|
1299
1305
|
options.split(//).uniq.each do |c| # FIX: this has a better home
|
@@ -1361,7 +1367,7 @@ module RubyParserStuff
|
|
1361
1367
|
end
|
1362
1368
|
|
1363
1369
|
def new_sclass val
|
1364
|
-
|
1370
|
+
(_, line), _, recv, in_def, _, in_single, body, _ = val
|
1365
1371
|
|
1366
1372
|
result = s(:sclass, recv)
|
1367
1373
|
|
@@ -1373,7 +1379,7 @@ module RubyParserStuff
|
|
1373
1379
|
end
|
1374
1380
|
end
|
1375
1381
|
|
1376
|
-
result.line =
|
1382
|
+
result.line = line
|
1377
1383
|
self.in_def = in_def
|
1378
1384
|
self.in_single = in_single
|
1379
1385
|
result
|