ruby_parser 3.20.0 → 3.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +1 -1
- data/History.rdoc +24 -0
- data/compare/normalize.rb +1 -0
- data/lib/ruby20_parser.rb +4266 -4283
- data/lib/ruby20_parser.y +50 -26
- data/lib/ruby21_parser.rb +4240 -4239
- data/lib/ruby21_parser.y +50 -26
- data/lib/ruby22_parser.rb +4288 -4289
- data/lib/ruby22_parser.y +50 -26
- data/lib/ruby23_parser.rb +4273 -4242
- data/lib/ruby23_parser.y +50 -26
- data/lib/ruby24_parser.rb +4278 -4297
- data/lib/ruby24_parser.y +50 -26
- data/lib/ruby25_parser.rb +4269 -4288
- data/lib/ruby25_parser.y +50 -26
- data/lib/ruby26_parser.rb +4269 -4288
- data/lib/ruby26_parser.y +50 -26
- data/lib/ruby27_parser.rb +4172 -4205
- data/lib/ruby27_parser.y +50 -26
- data/lib/ruby30_parser.rb +6028 -5970
- data/lib/ruby30_parser.y +135 -86
- data/lib/ruby31_parser.rb +6194 -6183
- data/lib/ruby31_parser.y +136 -87
- data/lib/ruby32_parser.rb +6290 -6327
- data/lib/ruby32_parser.y +157 -107
- data/lib/ruby3_parser.yy +172 -87
- data/lib/ruby_lexer.rb +15 -7
- data/lib/ruby_lexer.rex.rb +1 -1
- data/lib/ruby_lexer_strings.rb +2 -2
- data/lib/ruby_parser.yy +50 -26
- data/lib/ruby_parser_extras.rb +25 -19
- data/test/test_ruby_parser.rb +195 -7
- data/tools/munge.rb +8 -2
- data/tools/ripper.rb +14 -12
- data.tar.gz.sig +0 -0
- metadata +3 -3
- metadata.gz.sig +0 -0
data/lib/ruby_lexer.rb
CHANGED
@@ -117,7 +117,7 @@ class RubyLexer
|
|
117
117
|
|
118
118
|
def initialize _ = nil
|
119
119
|
@lex_state = nil # remove one warning under $DEBUG
|
120
|
-
|
120
|
+
@lex_state = EXPR_NONE
|
121
121
|
|
122
122
|
self.cond = RubyParserStuff::StackState.new(:cond, $DEBUG)
|
123
123
|
self.cmdarg = RubyParserStuff::StackState.new(:cmdarg, $DEBUG)
|
@@ -271,6 +271,8 @@ class RubyLexer
|
|
271
271
|
nil # TODO
|
272
272
|
end
|
273
273
|
|
274
|
+
# TODO: make all tXXXX terminals include lexer.lineno ... enforce it somehow?
|
275
|
+
|
274
276
|
def process_brace_close text
|
275
277
|
case matched
|
276
278
|
when "}" then
|
@@ -353,8 +355,14 @@ class RubyLexer
|
|
353
355
|
end
|
354
356
|
end
|
355
357
|
|
356
|
-
def process_dots text
|
357
|
-
|
358
|
+
def process_dots text # parse32.y:10216
|
359
|
+
is_beg = self.is_beg?
|
360
|
+
self.lex_state = EXPR_BEG
|
361
|
+
|
362
|
+
return result EXPR_ENDARG, :tBDOT3, text if
|
363
|
+
parser.in_argdef && text == "..." # TODO: version check?
|
364
|
+
|
365
|
+
tokens = ruby27plus? && is_beg ? BTOKENS : TOKENS
|
358
366
|
|
359
367
|
result EXPR_BEG, tokens[text], text
|
360
368
|
end
|
@@ -687,7 +695,7 @@ class RubyLexer
|
|
687
695
|
return process_token_keyword keyword if keyword
|
688
696
|
end
|
689
697
|
|
690
|
-
# matching: compare/
|
698
|
+
# matching: compare/parse32.y:9031
|
691
699
|
state = if lex_state =~ EXPR_BEG_ANY|EXPR_ARG_ANY|EXPR_DOT then
|
692
700
|
cmd_state ? EXPR_CMDARG : EXPR_ARG
|
693
701
|
elsif lex_state =~ EXPR_FNAME then
|
@@ -709,7 +717,7 @@ class RubyLexer
|
|
709
717
|
end
|
710
718
|
|
711
719
|
def process_token_keyword keyword
|
712
|
-
# matching MIDDLE of parse_ident in compare/
|
720
|
+
# matching MIDDLE of parse_ident in compare/parse32.y:9695
|
713
721
|
state = lex_state
|
714
722
|
|
715
723
|
return result(EXPR_ENDFN, keyword.id0, token) if lex_state =~ EXPR_FNAME
|
@@ -718,7 +726,7 @@ class RubyLexer
|
|
718
726
|
self.command_start = true if lex_state =~ EXPR_BEG
|
719
727
|
|
720
728
|
case
|
721
|
-
when keyword.id0 == :kDO then #
|
729
|
+
when keyword.id0 == :kDO then # parse32.y line 9712
|
722
730
|
case
|
723
731
|
when lambda_beginning? then
|
724
732
|
self.lpar_beg = nil # lambda_beginning? == FALSE in the body of "-> do ... end"
|
@@ -890,7 +898,7 @@ class RubyLexer
|
|
890
898
|
end
|
891
899
|
|
892
900
|
def inspect
|
893
|
-
return "
|
901
|
+
return "EXPR_NONE" if n.zero? # HACK?
|
894
902
|
|
895
903
|
names.map { |v, k| k if self =~ v }.
|
896
904
|
compact.
|
data/lib/ruby_lexer.rex.rb
CHANGED
data/lib/ruby_lexer_strings.rb
CHANGED
@@ -56,7 +56,7 @@ class RubyLexer
|
|
56
56
|
return :tSTRING_DVAR, matched
|
57
57
|
when scan(/#[{]/) then
|
58
58
|
self.command_start = true
|
59
|
-
return :tSTRING_DBEG, matched
|
59
|
+
return :tSTRING_DBEG, [matched, lineno]
|
60
60
|
when scan(/#/) then
|
61
61
|
string_buffer << "#"
|
62
62
|
end
|
@@ -415,7 +415,7 @@ class RubyLexer
|
|
415
415
|
return :tSTRING_DVAR, matched
|
416
416
|
when scan(/#[{]/) then
|
417
417
|
self.command_start = true
|
418
|
-
return :tSTRING_DBEG, matched
|
418
|
+
return :tSTRING_DBEG, [matched, lineno]
|
419
419
|
when scan(/#/) then
|
420
420
|
# do nothing but swallow
|
421
421
|
end
|
data/lib/ruby_parser.yy
CHANGED
@@ -253,6 +253,7 @@ rule
|
|
253
253
|
| lhs tEQL mrhs
|
254
254
|
{
|
255
255
|
lhs, _, rhs = val
|
256
|
+
|
256
257
|
result = new_assign lhs, s(:svalue, rhs).line(rhs.line)
|
257
258
|
}
|
258
259
|
#if V == 20
|
@@ -1107,8 +1108,10 @@ rule
|
|
1107
1108
|
|
1108
1109
|
paren_args: tLPAREN2 opt_call_args rparen
|
1109
1110
|
{
|
1110
|
-
_, args, _ = val
|
1111
|
+
_, args, (_, line_max) = val
|
1112
|
+
|
1111
1113
|
result = args
|
1114
|
+
result.line_max = line_max if args
|
1112
1115
|
}
|
1113
1116
|
#if V >= 27
|
1114
1117
|
| tLPAREN2 args tCOMMA args_forward rparen
|
@@ -1290,7 +1293,6 @@ rule
|
|
1290
1293
|
| k_begin
|
1291
1294
|
{
|
1292
1295
|
lexer.cmdarg.push false
|
1293
|
-
result = self.lexer.lineno
|
1294
1296
|
}
|
1295
1297
|
bodystmt k_end
|
1296
1298
|
{
|
@@ -1335,12 +1337,14 @@ rule
|
|
1335
1337
|
{
|
1336
1338
|
result = wrap :colon3, val[1]
|
1337
1339
|
}
|
1338
|
-
| tLBRACK { result = lexer.lineno } aref_args
|
1340
|
+
| tLBRACK { result = lexer.lineno } aref_args rbracket
|
1339
1341
|
{
|
1340
|
-
_, line, args, _ = val
|
1342
|
+
_, line, args, (_, line_max) = val
|
1343
|
+
|
1341
1344
|
result = args || s(:array)
|
1342
1345
|
result.sexp_type = :array # aref_args is :args
|
1343
1346
|
result.line line
|
1347
|
+
result.line_max = line_max
|
1344
1348
|
}
|
1345
1349
|
| tLBRACE
|
1346
1350
|
{
|
@@ -1453,9 +1457,6 @@ rule
|
|
1453
1457
|
result = new_for iter, var, body
|
1454
1458
|
}
|
1455
1459
|
| k_class
|
1456
|
-
{
|
1457
|
-
result = self.lexer.lineno
|
1458
|
-
}
|
1459
1460
|
cpath superclass
|
1460
1461
|
{
|
1461
1462
|
if (self.in_def || self.in_single > 0) then
|
@@ -1470,9 +1471,6 @@ rule
|
|
1470
1471
|
self.lexer.ignore_body_comments
|
1471
1472
|
}
|
1472
1473
|
| k_class tLSHFT
|
1473
|
-
{
|
1474
|
-
result = self.lexer.lineno
|
1475
|
-
}
|
1476
1474
|
expr
|
1477
1475
|
{
|
1478
1476
|
result = self.in_def
|
@@ -1491,9 +1489,6 @@ rule
|
|
1491
1489
|
self.lexer.ignore_body_comments
|
1492
1490
|
}
|
1493
1491
|
| k_module
|
1494
|
-
{
|
1495
|
-
result = self.lexer.lineno
|
1496
|
-
}
|
1497
1492
|
cpath
|
1498
1493
|
{
|
1499
1494
|
yyerror "module definition in method body" if
|
@@ -1942,13 +1937,19 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
1942
1937
|
{
|
1943
1938
|
call, args = val
|
1944
1939
|
|
1945
|
-
result = call
|
1940
|
+
result = call
|
1941
|
+
|
1942
|
+
if args then
|
1943
|
+
call.concat args.sexp_body
|
1944
|
+
result.line_max = args.line_max
|
1945
|
+
end
|
1946
1946
|
}
|
1947
1947
|
| primary_value call_op operation2 opt_paren_args
|
1948
1948
|
{
|
1949
|
-
recv, call_op, (op,
|
1949
|
+
recv, call_op, (op, op_line), args = val
|
1950
1950
|
|
1951
1951
|
result = new_call recv, op.to_sym, args, call_op
|
1952
|
+
result.line_max = op_line unless args
|
1952
1953
|
}
|
1953
1954
|
| primary_value tCOLON2 operation2 paren_args
|
1954
1955
|
{
|
@@ -2674,15 +2675,17 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2674
2675
|
|
2675
2676
|
words: tWORDS_BEG tSPACE tSTRING_END
|
2676
2677
|
{
|
2677
|
-
(_, line), _, _ = val
|
2678
|
+
(_, line), _, (_, line_max) = val
|
2678
2679
|
|
2679
2680
|
result = s(:array).line line
|
2681
|
+
result.line_max = line_max
|
2680
2682
|
}
|
2681
2683
|
| tWORDS_BEG word_list tSTRING_END
|
2682
2684
|
{
|
2683
|
-
(_, line), list, _ = val
|
2685
|
+
(_, line), list, (_, line_max) = val
|
2684
2686
|
|
2685
2687
|
result = list.line line
|
2688
|
+
result.line_max = line_max
|
2686
2689
|
}
|
2687
2690
|
|
2688
2691
|
word_list: none
|
@@ -2702,15 +2705,17 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2702
2705
|
|
2703
2706
|
symbols: tSYMBOLS_BEG tSPACE tSTRING_END
|
2704
2707
|
{
|
2705
|
-
(_, line), _, _ = val
|
2708
|
+
(_, line), _, (_, line_max) = val
|
2706
2709
|
|
2707
2710
|
result = s(:array).line line
|
2711
|
+
result.line_max = line_max
|
2708
2712
|
}
|
2709
2713
|
| tSYMBOLS_BEG symbol_list tSTRING_END
|
2710
2714
|
{
|
2711
|
-
(_, line), list, _, = val
|
2712
|
-
|
2713
|
-
result = list
|
2715
|
+
(_, line), list, (_, line_max), = val
|
2716
|
+
|
2717
|
+
result = list.line line
|
2718
|
+
result.line_max = line_max
|
2714
2719
|
}
|
2715
2720
|
|
2716
2721
|
symbol_list: none
|
@@ -2725,28 +2730,32 @@ opt_block_args_tail: tCOMMA block_args_tail
|
|
2725
2730
|
|
2726
2731
|
qwords: tQWORDS_BEG tSPACE tSTRING_END
|
2727
2732
|
{
|
2728
|
-
(_, line), _, _ = val
|
2733
|
+
(_, line), _, (_, line_max) = val
|
2729
2734
|
|
2730
2735
|
result = s(:array).line line
|
2736
|
+
result.line_max = line_max
|
2731
2737
|
}
|
2732
2738
|
| tQWORDS_BEG qword_list tSTRING_END
|
2733
2739
|
{
|
2734
|
-
(_, line), list, _ = val
|
2740
|
+
(_, line), list, (_, line_max) = val
|
2735
2741
|
|
2736
2742
|
result = list.line line
|
2743
|
+
result.line_max = line_max
|
2737
2744
|
}
|
2738
2745
|
|
2739
2746
|
qsymbols: tQSYMBOLS_BEG tSPACE tSTRING_END
|
2740
2747
|
{
|
2741
|
-
(_, line), _, _ = val
|
2748
|
+
(_, line), _, (_, line_max) = val
|
2742
2749
|
|
2743
2750
|
result = s(:array).line line
|
2751
|
+
result.line_max = line_max
|
2744
2752
|
}
|
2745
2753
|
| tQSYMBOLS_BEG qsym_list tSTRING_END
|
2746
2754
|
{
|
2747
|
-
(_, line), list, _ = val
|
2755
|
+
(_, line), list, (_, line_max) = val
|
2748
2756
|
|
2749
2757
|
result = list.line line
|
2758
|
+
result.line_max = line_max
|
2750
2759
|
}
|
2751
2760
|
|
2752
2761
|
qword_list: none
|
@@ -3197,7 +3206,14 @@ keyword_variable: kNIL { result = s(:nil).line lexer.lineno }
|
|
3197
3206
|
result = s(:args, list).line list.line
|
3198
3207
|
end
|
3199
3208
|
|
3200
|
-
|
3209
|
+
if Sexp === item then
|
3210
|
+
line_max = item.line_max
|
3211
|
+
else
|
3212
|
+
item, line_max = item
|
3213
|
+
end
|
3214
|
+
|
3215
|
+
result << item
|
3216
|
+
result.line_max = line_max
|
3201
3217
|
}
|
3202
3218
|
|
3203
3219
|
#if V == 20
|
@@ -3449,7 +3465,15 @@ keyword_variable: kNIL { result = s(:nil).line lexer.lineno }
|
|
3449
3465
|
opt_terms: | terms
|
3450
3466
|
opt_nl: | tNL
|
3451
3467
|
rparen: opt_nl tRPAREN
|
3468
|
+
{
|
3469
|
+
_, close = val
|
3470
|
+
result = [close, lexer.lineno]
|
3471
|
+
}
|
3452
3472
|
rbracket: opt_nl tRBRACK
|
3473
|
+
{
|
3474
|
+
_, close = val
|
3475
|
+
result = [close, lexer.lineno]
|
3476
|
+
}
|
3453
3477
|
#if V >= 27
|
3454
3478
|
rbrace: opt_nl tRCURLY
|
3455
3479
|
#endif
|
data/lib/ruby_parser_extras.rb
CHANGED
@@ -18,7 +18,7 @@ class Sexp
|
|
18
18
|
end
|
19
19
|
|
20
20
|
##
|
21
|
-
# Returns the
|
21
|
+
# Returns the minimum line number of the children of self.
|
22
22
|
|
23
23
|
def line_min
|
24
24
|
@line_min ||= [self.deep_each.map(&:line).min, self.line].compact.min
|
@@ -30,9 +30,9 @@ class Sexp
|
|
30
30
|
end
|
31
31
|
|
32
32
|
module RubyParserStuff
|
33
|
-
VERSION = "3.20.
|
33
|
+
VERSION = "3.20.3"
|
34
34
|
|
35
|
-
attr_accessor :lexer, :in_def, :in_single, :file
|
35
|
+
attr_accessor :lexer, :in_def, :in_single, :file, :in_argdef
|
36
36
|
attr_accessor :in_kwarg
|
37
37
|
attr_reader :env, :comments
|
38
38
|
|
@@ -122,6 +122,7 @@ module RubyParserStuff
|
|
122
122
|
self.lexer = RubyLexer.new v && v.to_i
|
123
123
|
self.lexer.parser = self
|
124
124
|
self.in_kwarg = false
|
125
|
+
self.in_argdef = false
|
125
126
|
|
126
127
|
@env = RubyParserStuff::Environment.new
|
127
128
|
@comments = []
|
@@ -153,6 +154,7 @@ module RubyParserStuff
|
|
153
154
|
result.line lexer.lineno
|
154
155
|
else
|
155
156
|
result.line ss.first.line
|
157
|
+
result.line_max = ss.first.line_max
|
156
158
|
end
|
157
159
|
|
158
160
|
args.each do |arg|
|
@@ -330,9 +332,8 @@ module RubyParserStuff
|
|
330
332
|
end
|
331
333
|
|
332
334
|
args.each do |arg|
|
333
|
-
|
334
|
-
|
335
|
-
end
|
335
|
+
# ruby 3.0+ TODO: next if arg in [String, Integer] # eg ["(", 1]
|
336
|
+
next if arg.class == Array && arg.map(&:class) == [String, Integer]
|
336
337
|
|
337
338
|
case arg
|
338
339
|
when Sexp then
|
@@ -794,6 +795,7 @@ module RubyParserStuff
|
|
794
795
|
case lhs.sexp_type
|
795
796
|
when :lasgn, :iasgn, :cdecl, :cvdecl, :gasgn, :cvasgn, :attrasgn, :safe_attrasgn then
|
796
797
|
lhs << rhs
|
798
|
+
lhs.line_max = rhs.line_max
|
797
799
|
when :const then
|
798
800
|
lhs.sexp_type = :cdecl
|
799
801
|
lhs << rhs
|
@@ -823,10 +825,10 @@ module RubyParserStuff
|
|
823
825
|
end
|
824
826
|
|
825
827
|
def new_begin val
|
826
|
-
_,
|
828
|
+
(_, line), _, body, _ = val
|
827
829
|
|
828
830
|
result = body ? s(:begin, body) : s(:nil)
|
829
|
-
result.line
|
831
|
+
result.line line
|
830
832
|
|
831
833
|
result
|
832
834
|
end
|
@@ -885,12 +887,13 @@ module RubyParserStuff
|
|
885
887
|
# TODO: need a test with f(&b) to produce block_pass
|
886
888
|
# TODO: need a test with f(&b) { } to produce warning
|
887
889
|
|
888
|
-
if args
|
890
|
+
if args then
|
889
891
|
if ARG_TYPES[args.sexp_type] then
|
890
892
|
result.concat args.sexp_body
|
891
893
|
else
|
892
894
|
result << args
|
893
895
|
end
|
896
|
+
result.line_max = args.line_max
|
894
897
|
end
|
895
898
|
|
896
899
|
# line = result.grep(Sexp).map(&:line).compact.min
|
@@ -926,8 +929,7 @@ module RubyParserStuff
|
|
926
929
|
end
|
927
930
|
|
928
931
|
def new_class val
|
929
|
-
|
930
|
-
line, path, superclass, body = val[1], val[2], val[3], val[5]
|
932
|
+
(_, line), path, superclass, _, body, (_, line_max) = val
|
931
933
|
|
932
934
|
path = path.first if path.instance_of? Array
|
933
935
|
|
@@ -942,6 +944,7 @@ module RubyParserStuff
|
|
942
944
|
end
|
943
945
|
|
944
946
|
result.line = line
|
947
|
+
result.line_max = line_max
|
945
948
|
result.comments = self.comments.pop
|
946
949
|
result
|
947
950
|
end
|
@@ -970,13 +973,14 @@ module RubyParserStuff
|
|
970
973
|
end
|
971
974
|
|
972
975
|
def new_defn val
|
973
|
-
_, (name, line), in_def, args, body, _ = val
|
976
|
+
_, (name, line), in_def, args, body, (_, line_max) = val
|
974
977
|
|
975
978
|
body ||= s(:nil).line line
|
976
979
|
|
977
980
|
args.line line
|
978
981
|
|
979
982
|
result = s(:defn, name.to_sym, args).line line
|
983
|
+
result.line_max = line_max
|
980
984
|
|
981
985
|
if body.sexp_type == :block then
|
982
986
|
result.push(*body.sexp_body)
|
@@ -1033,13 +1037,14 @@ module RubyParserStuff
|
|
1033
1037
|
end
|
1034
1038
|
|
1035
1039
|
def new_defs val
|
1036
|
-
_, recv, (name, line), in_def, args, body, _ = val
|
1040
|
+
_, recv, (name, line), in_def, args, body, (_, line_max) = val
|
1037
1041
|
|
1038
1042
|
body ||= s(:nil).line line
|
1039
1043
|
|
1040
1044
|
args.line line
|
1041
1045
|
|
1042
1046
|
result = s(:defs, recv, name.to_sym, args).line line
|
1047
|
+
result.line_max = line_max
|
1043
1048
|
|
1044
1049
|
# TODO: remove_begin
|
1045
1050
|
# TODO: reduce_nodes
|
@@ -1204,12 +1209,12 @@ module RubyParserStuff
|
|
1204
1209
|
end
|
1205
1210
|
|
1206
1211
|
def new_module val
|
1207
|
-
|
1208
|
-
line, path, body = val[1], val[2], val[4]
|
1212
|
+
(_, line_min), path, _, body, (_, line_max) = val
|
1209
1213
|
|
1210
1214
|
path = path.first if path.instance_of? Array
|
1211
1215
|
|
1212
|
-
result = s(:module, path).line
|
1216
|
+
result = s(:module, path).line line_min
|
1217
|
+
result.line_max = line_max
|
1213
1218
|
|
1214
1219
|
if body then # REFACTOR?
|
1215
1220
|
if body.sexp_type == :block then
|
@@ -1291,9 +1296,10 @@ module RubyParserStuff
|
|
1291
1296
|
end
|
1292
1297
|
|
1293
1298
|
def new_regexp val
|
1294
|
-
(_, line), node, (options,
|
1299
|
+
(_, line), node, (options, line_max) = val
|
1295
1300
|
|
1296
1301
|
node ||= s(:str, "").line line
|
1302
|
+
node.line_max = line_max
|
1297
1303
|
|
1298
1304
|
o, k = 0, nil
|
1299
1305
|
options.split(//).uniq.each do |c| # FIX: this has a better home
|
@@ -1361,7 +1367,7 @@ module RubyParserStuff
|
|
1361
1367
|
end
|
1362
1368
|
|
1363
1369
|
def new_sclass val
|
1364
|
-
|
1370
|
+
(_, line), _, recv, in_def, _, in_single, body, _ = val
|
1365
1371
|
|
1366
1372
|
result = s(:sclass, recv)
|
1367
1373
|
|
@@ -1373,7 +1379,7 @@ module RubyParserStuff
|
|
1373
1379
|
end
|
1374
1380
|
end
|
1375
1381
|
|
1376
|
-
result.line =
|
1382
|
+
result.line = line
|
1377
1383
|
self.in_def = in_def
|
1378
1384
|
self.in_single = in_single
|
1379
1385
|
result
|