parser 2.6.5.0 → 2.7.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -0
- data/CHANGELOG.md +52 -0
- data/README.md +8 -1
- data/doc/AST_FORMAT.md +384 -20
- data/lib/parser.rb +3 -1
- data/lib/parser/ast/processor.rb +19 -0
- data/lib/parser/base.rb +19 -0
- data/lib/parser/builders/default.rb +250 -12
- data/lib/parser/context.rb +4 -0
- data/lib/parser/current.rb +4 -4
- data/lib/parser/current_arg_stack.rb +43 -0
- data/lib/parser/lexer.rl +98 -95
- data/lib/parser/lexer/dedenter.rb +52 -49
- data/lib/parser/{lexer/max_numparam_stack.rb → max_numparam_stack.rb} +10 -4
- data/lib/parser/messages.rb +35 -29
- data/lib/parser/meta.rb +7 -2
- data/lib/parser/ruby27.y +495 -35
- data/lib/parser/source/tree_rewriter/action.rb +2 -2
- data/lib/parser/static_environment.rb +10 -0
- data/lib/parser/variables_stack.rb +32 -0
- data/lib/parser/version.rb +1 -1
- data/parser.gemspec +7 -0
- data/test/helper.rb +1 -0
- data/test/parse_helper.rb +3 -0
- data/test/test_lexer.rb +7 -66
- data/test/test_parser.rb +1831 -118
- data/test/test_source_comment_associator.rb +20 -20
- metadata +11 -5
data/lib/parser.rb
CHANGED
@@ -67,13 +67,15 @@ module Parser
|
|
67
67
|
require 'parser/lexer/literal'
|
68
68
|
require 'parser/lexer/stack_state'
|
69
69
|
require 'parser/lexer/dedenter'
|
70
|
-
require 'parser/lexer/max_numparam_stack'
|
71
70
|
|
72
71
|
module Builders
|
73
72
|
require 'parser/builders/default'
|
74
73
|
end
|
75
74
|
|
76
75
|
require 'parser/context'
|
76
|
+
require 'parser/max_numparam_stack'
|
77
|
+
require 'parser/current_arg_stack'
|
78
|
+
require 'parser/variables_stack'
|
77
79
|
|
78
80
|
require 'parser/base'
|
79
81
|
|
data/lib/parser/ast/processor.rb
CHANGED
@@ -236,6 +236,21 @@ module Parser
|
|
236
236
|
alias on_preexe process_regular_node
|
237
237
|
alias on_postexe process_regular_node
|
238
238
|
|
239
|
+
alias on_case_match process_regular_node
|
240
|
+
alias on_in_match process_regular_node
|
241
|
+
alias on_in_pattern process_regular_node
|
242
|
+
alias on_if_guard process_regular_node
|
243
|
+
alias on_unless_guard process_regular_node
|
244
|
+
alias on_match_var process_variable_node
|
245
|
+
alias on_match_rest process_regular_node
|
246
|
+
alias on_pin process_regular_node
|
247
|
+
alias on_match_alt process_regular_node
|
248
|
+
alias on_match_as process_regular_node
|
249
|
+
alias on_array_pattern process_regular_node
|
250
|
+
alias on_array_pattern_with_tail process_regular_node
|
251
|
+
alias on_hash_pattern process_regular_node
|
252
|
+
alias on_const_pattern process_regular_node
|
253
|
+
|
239
254
|
# @private
|
240
255
|
def process_variable_node(node)
|
241
256
|
warn 'Parser::AST::Processor#process_variable_node is deprecated as a' \
|
@@ -259,6 +274,10 @@ module Parser
|
|
259
274
|
'Parser::AST::Processor#on_argument instead.'
|
260
275
|
on_argument(node)
|
261
276
|
end
|
277
|
+
|
278
|
+
def on_empty_else(node)
|
279
|
+
node
|
280
|
+
end
|
262
281
|
end
|
263
282
|
end
|
264
283
|
end
|
data/lib/parser/base.rb
CHANGED
@@ -114,6 +114,10 @@ module Parser
|
|
114
114
|
attr_reader :static_env
|
115
115
|
attr_reader :source_buffer
|
116
116
|
attr_reader :context
|
117
|
+
attr_reader :max_numparam_stack
|
118
|
+
attr_reader :current_arg_stack
|
119
|
+
attr_reader :pattern_variables
|
120
|
+
attr_reader :pattern_hash_keys
|
117
121
|
|
118
122
|
##
|
119
123
|
# @param [Parser::Builders::Default] builder The AST builder to use.
|
@@ -126,6 +130,18 @@ module Parser
|
|
126
130
|
# Stack that holds current parsing context
|
127
131
|
@context = Context.new
|
128
132
|
|
133
|
+
# Maximum numbered parameters stack
|
134
|
+
@max_numparam_stack = MaxNumparamStack.new
|
135
|
+
|
136
|
+
# Current argument names stack
|
137
|
+
@current_arg_stack = CurrentArgStack.new
|
138
|
+
|
139
|
+
# Stack of set of variables used in the current pattern
|
140
|
+
@pattern_variables = VariablesStack.new
|
141
|
+
|
142
|
+
# Stack of set of keys used in the current hash in pattern matchinig
|
143
|
+
@pattern_hash_keys = VariablesStack.new
|
144
|
+
|
129
145
|
@lexer = Lexer.new(version)
|
130
146
|
@lexer.diagnostics = @diagnostics
|
131
147
|
@lexer.static_env = @static_env
|
@@ -153,6 +169,9 @@ module Parser
|
|
153
169
|
@lexer.reset
|
154
170
|
@static_env.reset
|
155
171
|
@context.reset
|
172
|
+
@current_arg_stack.reset
|
173
|
+
@pattern_variables.reset
|
174
|
+
@pattern_hash_keys.reset
|
156
175
|
|
157
176
|
self
|
158
177
|
end
|
@@ -453,11 +453,6 @@ module Parser
|
|
453
453
|
variable_map(token))
|
454
454
|
end
|
455
455
|
|
456
|
-
def numparam(token)
|
457
|
-
n(:numparam, [ value(token).to_i ],
|
458
|
-
variable_map(token))
|
459
|
-
end
|
460
|
-
|
461
456
|
def back_ref(token)
|
462
457
|
n(:back_ref, [ value(token).to_sym ],
|
463
458
|
token_map(token))
|
@@ -498,6 +493,11 @@ module Parser
|
|
498
493
|
name, = *node
|
499
494
|
|
500
495
|
if @parser.static_env.declared?(name)
|
496
|
+
if name.to_s == parser.current_arg_stack.top
|
497
|
+
diagnostic :error, :circular_argument_reference,
|
498
|
+
{ :var_name => name.to_s }, node.loc.expression
|
499
|
+
end
|
500
|
+
|
501
501
|
node.updated(:lvar)
|
502
502
|
else
|
503
503
|
name, = *node
|
@@ -556,6 +556,9 @@ module Parser
|
|
556
556
|
|
557
557
|
when :ident
|
558
558
|
name, = *node
|
559
|
+
|
560
|
+
check_assignment_to_numparam(node)
|
561
|
+
|
559
562
|
@parser.static_env.declare(name)
|
560
563
|
|
561
564
|
node.updated(:lvasgn)
|
@@ -688,6 +691,10 @@ module Parser
|
|
688
691
|
n(:numargs, [ max_numparam ], nil)
|
689
692
|
end
|
690
693
|
|
694
|
+
def forward_args(begin_t, dots_t, end_t)
|
695
|
+
n(:forward_args, [], collection_map(begin_t, token_map(dots_t), end_t))
|
696
|
+
end
|
697
|
+
|
691
698
|
def arg(name_t)
|
692
699
|
n(:arg, [ value(name_t).to_sym ],
|
693
700
|
variable_map(name_t))
|
@@ -837,6 +844,10 @@ module Parser
|
|
837
844
|
end
|
838
845
|
end
|
839
846
|
|
847
|
+
def forwarded_args(dots_t)
|
848
|
+
n(:forwarded_args, [], token_map(dots_t))
|
849
|
+
end
|
850
|
+
|
840
851
|
def call_method(receiver, dot_t, selector_t,
|
841
852
|
lparen_t=nil, args=[], rparen_t=nil)
|
842
853
|
type = call_type_for_dot(dot_t)
|
@@ -866,11 +877,10 @@ module Parser
|
|
866
877
|
end
|
867
878
|
|
868
879
|
last_arg = call_args.last
|
869
|
-
if last_arg && last_arg.type == :block_pass
|
880
|
+
if last_arg && (last_arg.type == :block_pass || last_arg.type == :forwarded_args)
|
870
881
|
diagnostic :error, :block_and_blockarg, nil, last_arg.loc.expression, [loc(begin_t)]
|
871
882
|
end
|
872
883
|
|
873
|
-
|
874
884
|
if args.type == :numargs
|
875
885
|
block_type = :numblock
|
876
886
|
args = args.children[0]
|
@@ -1006,11 +1016,6 @@ module Parser
|
|
1006
1016
|
end
|
1007
1017
|
end
|
1008
1018
|
|
1009
|
-
def method_ref(receiver, dot_t, selector_t)
|
1010
|
-
n(:meth_ref, [ receiver, value(selector_t).to_sym ],
|
1011
|
-
send_map(receiver, dot_t, selector_t, nil, [], nil))
|
1012
|
-
end
|
1013
|
-
|
1014
1019
|
#
|
1015
1020
|
# Control flow
|
1016
1021
|
#
|
@@ -1203,6 +1208,193 @@ module Parser
|
|
1203
1208
|
end
|
1204
1209
|
end
|
1205
1210
|
|
1211
|
+
#
|
1212
|
+
# PATTERN MATCHING
|
1213
|
+
#
|
1214
|
+
|
1215
|
+
def case_match(case_t, expr, in_bodies, else_t, else_body, end_t)
|
1216
|
+
else_body = n(:empty_else, nil, token_map(else_t)) if else_t && !else_body
|
1217
|
+
n(:case_match, [ expr, *(in_bodies << else_body)],
|
1218
|
+
condition_map(case_t, expr, nil, nil, else_t, else_body, end_t))
|
1219
|
+
end
|
1220
|
+
|
1221
|
+
def in_match(lhs, in_t, rhs)
|
1222
|
+
n(:in_match, [lhs, rhs],
|
1223
|
+
binary_op_map(lhs, in_t, rhs))
|
1224
|
+
end
|
1225
|
+
|
1226
|
+
def in_pattern(in_t, pattern, guard, then_t, body)
|
1227
|
+
children = [pattern, guard, body]
|
1228
|
+
n(:in_pattern, children,
|
1229
|
+
keyword_map(in_t, then_t, children.compact, nil))
|
1230
|
+
end
|
1231
|
+
|
1232
|
+
def if_guard(if_t, if_body)
|
1233
|
+
n(:if_guard, [if_body], guard_map(if_t, if_body))
|
1234
|
+
end
|
1235
|
+
|
1236
|
+
def unless_guard(unless_t, unless_body)
|
1237
|
+
n(:unless_guard, [unless_body], guard_map(unless_t, unless_body))
|
1238
|
+
end
|
1239
|
+
|
1240
|
+
def match_var(name_t)
|
1241
|
+
name = value(name_t).to_sym
|
1242
|
+
|
1243
|
+
check_duplicate_pattern_variable(name, loc(name_t))
|
1244
|
+
@parser.static_env.declare(name)
|
1245
|
+
|
1246
|
+
n(:match_var, [ name ],
|
1247
|
+
variable_map(name_t))
|
1248
|
+
end
|
1249
|
+
|
1250
|
+
def match_hash_var(name_t)
|
1251
|
+
name = value(name_t).to_sym
|
1252
|
+
|
1253
|
+
expr_l = loc(name_t)
|
1254
|
+
name_l = expr_l.adjust(end_pos: -1)
|
1255
|
+
|
1256
|
+
check_duplicate_pattern_variable(name, name_l)
|
1257
|
+
@parser.static_env.declare(name)
|
1258
|
+
|
1259
|
+
n(:match_var, [ name ],
|
1260
|
+
Source::Map::Variable.new(name_l, expr_l))
|
1261
|
+
end
|
1262
|
+
|
1263
|
+
def match_hash_var_from_str(begin_t, strings, end_t)
|
1264
|
+
if strings.length > 1
|
1265
|
+
diagnostic :error, :pm_interp_in_var_name, nil, loc(begin_t).join(loc(end_t))
|
1266
|
+
end
|
1267
|
+
|
1268
|
+
string = strings[0]
|
1269
|
+
|
1270
|
+
case string.type
|
1271
|
+
when :str
|
1272
|
+
# MRI supports plain strings in hash pattern matching
|
1273
|
+
name, = *string
|
1274
|
+
name_l = string.loc.expression
|
1275
|
+
|
1276
|
+
check_lvar_name(name, name_l)
|
1277
|
+
check_duplicate_pattern_variable(name, name_l)
|
1278
|
+
|
1279
|
+
@parser.static_env.declare(name)
|
1280
|
+
|
1281
|
+
if (begin_l = string.loc.begin)
|
1282
|
+
# exclude beginning of the string from the location of the variable
|
1283
|
+
name_l = name_l.adjust(begin_pos: begin_l.length)
|
1284
|
+
end
|
1285
|
+
|
1286
|
+
if (end_l = string.loc.end)
|
1287
|
+
# exclude end of the string from the location of the variable
|
1288
|
+
name_l = name_l.adjust(end_pos: -end_l.length)
|
1289
|
+
end
|
1290
|
+
|
1291
|
+
expr_l = loc(begin_t).join(string.loc.expression).join(loc(end_t))
|
1292
|
+
n(:match_var, [ name.to_sym ],
|
1293
|
+
Source::Map::Variable.new(name_l, expr_l))
|
1294
|
+
when :begin
|
1295
|
+
match_hash_var_from_str(begin_t, string.children, end_t)
|
1296
|
+
end
|
1297
|
+
end
|
1298
|
+
|
1299
|
+
def match_rest(star_t, name_t = nil)
|
1300
|
+
if name_t.nil?
|
1301
|
+
n0(:match_rest,
|
1302
|
+
unary_op_map(star_t))
|
1303
|
+
else
|
1304
|
+
name = match_var(name_t)
|
1305
|
+
n(:match_rest, [ name ],
|
1306
|
+
unary_op_map(star_t, name))
|
1307
|
+
end
|
1308
|
+
end
|
1309
|
+
|
1310
|
+
def hash_pattern(lbrace_t, kwargs, rbrace_t)
|
1311
|
+
args = check_duplicate_args(kwargs)
|
1312
|
+
n(:hash_pattern, args,
|
1313
|
+
collection_map(lbrace_t, args, rbrace_t))
|
1314
|
+
end
|
1315
|
+
|
1316
|
+
def array_pattern(lbrack_t, elements, rbrack_t)
|
1317
|
+
return n(:array_pattern, nil, collection_map(lbrack_t, [], rbrack_t)) if elements.nil?
|
1318
|
+
|
1319
|
+
trailing_comma = false
|
1320
|
+
|
1321
|
+
elements = elements.map do |element|
|
1322
|
+
if element.type == :match_with_trailing_comma
|
1323
|
+
trailing_comma = true
|
1324
|
+
element.children.first
|
1325
|
+
else
|
1326
|
+
trailing_comma = false
|
1327
|
+
element
|
1328
|
+
end
|
1329
|
+
end
|
1330
|
+
|
1331
|
+
node_type = trailing_comma ? :array_pattern_with_tail : :array_pattern
|
1332
|
+
n(node_type, elements,
|
1333
|
+
collection_map(lbrack_t, elements, rbrack_t))
|
1334
|
+
end
|
1335
|
+
|
1336
|
+
def match_with_trailing_comma(match)
|
1337
|
+
n(:match_with_trailing_comma, [ match ], nil)
|
1338
|
+
end
|
1339
|
+
|
1340
|
+
def const_pattern(const, ldelim_t, pattern, rdelim_t)
|
1341
|
+
n(:const_pattern, [const, pattern],
|
1342
|
+
collection_map(ldelim_t, [pattern], rdelim_t))
|
1343
|
+
end
|
1344
|
+
|
1345
|
+
def pin(pin_t, var)
|
1346
|
+
n(:pin, [ var ],
|
1347
|
+
send_unary_op_map(pin_t, var))
|
1348
|
+
end
|
1349
|
+
|
1350
|
+
def match_alt(left, pipe_t, right)
|
1351
|
+
source_map = binary_op_map(left, pipe_t, right)
|
1352
|
+
|
1353
|
+
n(:match_alt, [ left, right ],
|
1354
|
+
source_map)
|
1355
|
+
end
|
1356
|
+
|
1357
|
+
def match_as(value, assoc_t, as)
|
1358
|
+
source_map = binary_op_map(value, assoc_t, as)
|
1359
|
+
|
1360
|
+
n(:match_as, [ value, as ],
|
1361
|
+
source_map)
|
1362
|
+
end
|
1363
|
+
|
1364
|
+
def match_nil_pattern(dstar_t, nil_t)
|
1365
|
+
n0(:match_nil_pattern,
|
1366
|
+
arg_prefix_map(dstar_t, nil_t))
|
1367
|
+
end
|
1368
|
+
|
1369
|
+
def match_pair(label_type, label, value)
|
1370
|
+
if label_type == :label
|
1371
|
+
check_duplicate_pattern_key(label[0], label[1])
|
1372
|
+
pair_keyword(label, value)
|
1373
|
+
else
|
1374
|
+
begin_t, parts, end_t = label
|
1375
|
+
label_loc = loc(begin_t).join(loc(end_t))
|
1376
|
+
|
1377
|
+
# quoted label like "label": value
|
1378
|
+
if (var_name = static_string(parts))
|
1379
|
+
check_duplicate_pattern_key(var_name, label_loc)
|
1380
|
+
else
|
1381
|
+
diagnostic :error, :pm_interp_in_var_name, nil, label_loc
|
1382
|
+
end
|
1383
|
+
|
1384
|
+
pair_quoted(begin_t, parts, end_t, value)
|
1385
|
+
end
|
1386
|
+
end
|
1387
|
+
|
1388
|
+
def match_label(label_type, label)
|
1389
|
+
if label_type == :label
|
1390
|
+
match_hash_var(label)
|
1391
|
+
else
|
1392
|
+
# quoted label like "label": value
|
1393
|
+
begin_t, strings, end_t = label
|
1394
|
+
match_hash_var_from_str(begin_t, strings, end_t)
|
1395
|
+
end
|
1396
|
+
end
|
1397
|
+
|
1206
1398
|
private
|
1207
1399
|
|
1208
1400
|
#
|
@@ -1292,6 +1484,19 @@ module Parser
|
|
1292
1484
|
end
|
1293
1485
|
end
|
1294
1486
|
|
1487
|
+
def check_assignment_to_numparam(node)
|
1488
|
+
name = node.children[0].to_s
|
1489
|
+
|
1490
|
+
assigning_to_numparam =
|
1491
|
+
@parser.context.in_dynamic_block? &&
|
1492
|
+
name =~ /\A_([1-9])\z/ &&
|
1493
|
+
@parser.max_numparam_stack.has_numparams?
|
1494
|
+
|
1495
|
+
if assigning_to_numparam
|
1496
|
+
diagnostic :error, :cant_assign_to_numparam, { :name => name }, node.loc.expression
|
1497
|
+
end
|
1498
|
+
end
|
1499
|
+
|
1295
1500
|
def arg_name_collides?(this_name, that_name)
|
1296
1501
|
case @parser.version
|
1297
1502
|
when 18
|
@@ -1307,6 +1512,32 @@ module Parser
|
|
1307
1512
|
end
|
1308
1513
|
end
|
1309
1514
|
|
1515
|
+
def check_lvar_name(name, loc)
|
1516
|
+
if name =~ /\A[[[:lower:]]|_][[[:alnum:]]_]*\z/
|
1517
|
+
# OK
|
1518
|
+
else
|
1519
|
+
diagnostic :error, :lvar_name, { name: name }, loc
|
1520
|
+
end
|
1521
|
+
end
|
1522
|
+
|
1523
|
+
def check_duplicate_pattern_variable(name, loc)
|
1524
|
+
return if name.to_s.start_with?('_')
|
1525
|
+
|
1526
|
+
if @parser.pattern_variables.declared?(name)
|
1527
|
+
diagnostic :error, :duplicate_variable_name, { name: name.to_s }, loc
|
1528
|
+
end
|
1529
|
+
|
1530
|
+
@parser.pattern_variables.declare(name)
|
1531
|
+
end
|
1532
|
+
|
1533
|
+
def check_duplicate_pattern_key(name, loc)
|
1534
|
+
if @parser.pattern_hash_keys.declared?(name)
|
1535
|
+
diagnostic :error, :duplicate_pattern_key, { name: name.to_s }, loc
|
1536
|
+
end
|
1537
|
+
|
1538
|
+
@parser.pattern_hash_keys.declare(name)
|
1539
|
+
end
|
1540
|
+
|
1310
1541
|
#
|
1311
1542
|
# SOURCE MAPS
|
1312
1543
|
#
|
@@ -1655,6 +1886,13 @@ module Parser
|
|
1655
1886
|
begin_l.join(end_l))
|
1656
1887
|
end
|
1657
1888
|
|
1889
|
+
def guard_map(keyword_t, guard_body_e)
|
1890
|
+
keyword_l = loc(keyword_t)
|
1891
|
+
guard_body_l = guard_body_e.loc.expression
|
1892
|
+
|
1893
|
+
Source::Map::Keyword.new(keyword_l, nil, nil, keyword_l.join(guard_body_l))
|
1894
|
+
end
|
1895
|
+
|
1658
1896
|
#
|
1659
1897
|
# HELPERS
|
1660
1898
|
#
|
data/lib/parser/context.rb
CHANGED
data/lib/parser/current.rb
CHANGED
@@ -75,7 +75,7 @@ module Parser
|
|
75
75
|
CurrentRuby = Ruby26
|
76
76
|
|
77
77
|
when /^2\.7\./
|
78
|
-
current_version = '2.7.0
|
78
|
+
current_version = '2.7.0'
|
79
79
|
if RUBY_VERSION != current_version
|
80
80
|
warn_syntax_deviation 'parser/ruby27', current_version
|
81
81
|
end
|
@@ -85,8 +85,8 @@ module Parser
|
|
85
85
|
|
86
86
|
else # :nocov:
|
87
87
|
# Keep this in sync with released Ruby.
|
88
|
-
warn_syntax_deviation 'parser/
|
89
|
-
require 'parser/
|
90
|
-
CurrentRuby =
|
88
|
+
warn_syntax_deviation 'parser/ruby27', '2.7.x'
|
89
|
+
require 'parser/ruby27'
|
90
|
+
CurrentRuby = Ruby27
|
91
91
|
end
|
92
92
|
end
|