syntax_tree 5.3.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -1
- data/CHANGELOG.md +64 -1
- data/Gemfile.lock +2 -2
- data/README.md +28 -9
- data/Rakefile +12 -8
- data/bin/console +1 -0
- data/bin/whitequark +79 -0
- data/doc/changing_structure.md +16 -0
- data/lib/syntax_tree/basic_visitor.rb +44 -5
- data/lib/syntax_tree/cli.rb +2 -2
- data/lib/syntax_tree/dsl.rb +23 -11
- data/lib/syntax_tree/{visitor/field_visitor.rb → field_visitor.rb} +54 -55
- data/lib/syntax_tree/formatter.rb +1 -1
- data/lib/syntax_tree/index.rb +56 -54
- data/lib/syntax_tree/json_visitor.rb +55 -0
- data/lib/syntax_tree/language_server.rb +157 -2
- data/lib/syntax_tree/match_visitor.rb +120 -0
- data/lib/syntax_tree/mermaid.rb +177 -0
- data/lib/syntax_tree/mermaid_visitor.rb +69 -0
- data/lib/syntax_tree/{visitor/mutation_visitor.rb → mutation_visitor.rb} +27 -27
- data/lib/syntax_tree/node.rb +198 -107
- data/lib/syntax_tree/parser.rb +322 -118
- data/lib/syntax_tree/pretty_print_visitor.rb +83 -0
- data/lib/syntax_tree/reflection.rb +241 -0
- data/lib/syntax_tree/translation/parser.rb +3019 -0
- data/lib/syntax_tree/translation/rubocop_ast.rb +21 -0
- data/lib/syntax_tree/translation.rb +28 -0
- data/lib/syntax_tree/version.rb +1 -1
- data/lib/syntax_tree/with_scope.rb +244 -0
- data/lib/syntax_tree/yarv/basic_block.rb +53 -0
- data/lib/syntax_tree/yarv/calldata.rb +91 -0
- data/lib/syntax_tree/yarv/compiler.rb +110 -100
- data/lib/syntax_tree/yarv/control_flow_graph.rb +257 -0
- data/lib/syntax_tree/yarv/data_flow_graph.rb +338 -0
- data/lib/syntax_tree/yarv/decompiler.rb +1 -1
- data/lib/syntax_tree/yarv/disassembler.rb +104 -80
- data/lib/syntax_tree/yarv/instruction_sequence.rb +43 -18
- data/lib/syntax_tree/yarv/instructions.rb +203 -649
- data/lib/syntax_tree/yarv/legacy.rb +12 -24
- data/lib/syntax_tree/yarv/sea_of_nodes.rb +534 -0
- data/lib/syntax_tree/yarv.rb +18 -0
- data/lib/syntax_tree.rb +88 -56
- data/tasks/sorbet.rake +277 -0
- data/tasks/whitequark.rake +87 -0
- metadata +23 -11
- data/.gitmodules +0 -9
- data/lib/syntax_tree/language_server/inlay_hints.rb +0 -159
- data/lib/syntax_tree/visitor/environment.rb +0 -84
- data/lib/syntax_tree/visitor/json_visitor.rb +0 -55
- data/lib/syntax_tree/visitor/match_visitor.rb +0 -122
- data/lib/syntax_tree/visitor/pretty_print_visitor.rb +0 -85
- data/lib/syntax_tree/visitor/with_environment.rb +0 -140
data/lib/syntax_tree/parser.rb
CHANGED
@@ -256,11 +256,37 @@ module SyntaxTree
|
|
256
256
|
tokens[index] if index
|
257
257
|
end
|
258
258
|
|
259
|
+
def find_token_between(type, left, right)
|
260
|
+
bounds = left.location.end_char...right.location.start_char
|
261
|
+
index =
|
262
|
+
tokens.rindex do |token|
|
263
|
+
char = token.location.start_char
|
264
|
+
break if char < bounds.begin
|
265
|
+
|
266
|
+
token.is_a?(type) && bounds.cover?(char)
|
267
|
+
end
|
268
|
+
|
269
|
+
tokens[index] if index
|
270
|
+
end
|
271
|
+
|
259
272
|
def find_keyword(name)
|
260
273
|
index = tokens.rindex { |token| token.is_a?(Kw) && (token.name == name) }
|
261
274
|
tokens[index] if index
|
262
275
|
end
|
263
276
|
|
277
|
+
def find_keyword_between(name, left, right)
|
278
|
+
bounds = left.end_char...right.start_char
|
279
|
+
index =
|
280
|
+
tokens.rindex do |token|
|
281
|
+
char = token.location.start_char
|
282
|
+
break if char < bounds.begin
|
283
|
+
|
284
|
+
token.is_a?(Kw) && (token.name == name) && bounds.cover?(char)
|
285
|
+
end
|
286
|
+
|
287
|
+
tokens[index] if index
|
288
|
+
end
|
289
|
+
|
264
290
|
def find_operator(name)
|
265
291
|
index = tokens.rindex { |token| token.is_a?(Op) && (token.name == name) }
|
266
292
|
tokens[index] if index
|
@@ -348,6 +374,7 @@ module SyntaxTree
|
|
348
374
|
|
349
375
|
start_char = find_next_statement_start(lbrace.location.end_char)
|
350
376
|
statements.bind(
|
377
|
+
self,
|
351
378
|
start_char,
|
352
379
|
start_char - line_counts[lbrace.location.start_line - 1].start,
|
353
380
|
rbrace.location.start_char,
|
@@ -386,6 +413,7 @@ module SyntaxTree
|
|
386
413
|
|
387
414
|
start_char = find_next_statement_start(lbrace.location.end_char)
|
388
415
|
statements.bind(
|
416
|
+
self,
|
389
417
|
start_char,
|
390
418
|
start_char - line_counts[lbrace.location.start_line - 1].start,
|
391
419
|
rbrace.location.start_char,
|
@@ -640,13 +668,14 @@ module SyntaxTree
|
|
640
668
|
stack.pop
|
641
669
|
end
|
642
670
|
|
643
|
-
|
644
|
-
|
645
|
-
|
671
|
+
visit_methods do
|
672
|
+
def visit_var_ref(node)
|
673
|
+
node.pin(stack[-2], pins.shift)
|
674
|
+
end
|
646
675
|
end
|
647
676
|
|
648
677
|
def self.visit(node, tokens)
|
649
|
-
start_char = node.
|
678
|
+
start_char = node.start_char
|
650
679
|
allocated = []
|
651
680
|
|
652
681
|
tokens.reverse_each do |token|
|
@@ -670,18 +699,22 @@ module SyntaxTree
|
|
670
699
|
# (nil | Array[untyped]) posts
|
671
700
|
# ) -> AryPtn
|
672
701
|
def on_aryptn(constant, requireds, rest, posts)
|
673
|
-
|
702
|
+
lbracket = find_token(LBracket)
|
703
|
+
lbracket ||= find_token(LParen) if constant
|
674
704
|
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
705
|
+
rbracket = find_token(RBracket)
|
706
|
+
rbracket ||= find_token(RParen) if constant
|
707
|
+
|
708
|
+
parts = [constant, lbracket, *requireds, rest, *posts, rbracket].compact
|
709
|
+
|
710
|
+
# The location is going to be determined by the first part to the last
|
711
|
+
# part. This includes potential brackets.
|
712
|
+
location = parts[0].location.to(parts[-1].location)
|
713
|
+
|
714
|
+
# Now that we have the location calculated, we can remove the brackets
|
715
|
+
# from the list of tokens.
|
716
|
+
tokens.delete(lbracket) if lbracket
|
717
|
+
tokens.delete(rbracket) if rbracket
|
685
718
|
|
686
719
|
# If there is a plain *, then we're going to fix up the location of it
|
687
720
|
# here because it currently doesn't have anything to use for its precise
|
@@ -820,6 +853,7 @@ module SyntaxTree
|
|
820
853
|
end
|
821
854
|
|
822
855
|
bodystmt.bind(
|
856
|
+
self,
|
823
857
|
find_next_statement_start(keyword.location.end_char),
|
824
858
|
keyword.location.end_column,
|
825
859
|
end_location.end_char,
|
@@ -871,13 +905,34 @@ module SyntaxTree
|
|
871
905
|
# on_block_var: (Params params, (nil | Array[Ident]) locals) -> BlockVar
|
872
906
|
def on_block_var(params, locals)
|
873
907
|
index =
|
874
|
-
tokens.rindex
|
875
|
-
|
876
|
-
|
877
|
-
|
908
|
+
tokens.rindex { |node| node.is_a?(Op) && %w[| ||].include?(node.value) }
|
909
|
+
|
910
|
+
ending = tokens.delete_at(index)
|
911
|
+
beginning = ending.value == "||" ? ending : consume_operator(:|)
|
912
|
+
|
913
|
+
# If there are no parameters, then we didn't have anything to base the
|
914
|
+
# location information of off. Now that we have an opening of the
|
915
|
+
# block, we can correct this.
|
916
|
+
if params.empty?
|
917
|
+
start_line = params.location.start_line
|
918
|
+
start_char =
|
919
|
+
(
|
920
|
+
if beginning.value == "||"
|
921
|
+
beginning.location.start_char
|
922
|
+
else
|
923
|
+
find_next_statement_start(beginning.location.end_char)
|
924
|
+
end
|
925
|
+
)
|
878
926
|
|
879
|
-
|
880
|
-
|
927
|
+
location =
|
928
|
+
Location.fixed(
|
929
|
+
line: start_line,
|
930
|
+
char: start_char,
|
931
|
+
column: start_char - line_counts[start_line - 1].start
|
932
|
+
)
|
933
|
+
|
934
|
+
params = params.copy(location: location)
|
935
|
+
end
|
881
936
|
|
882
937
|
BlockVar.new(
|
883
938
|
params: params,
|
@@ -905,6 +960,14 @@ module SyntaxTree
|
|
905
960
|
# (nil | Ensure) ensure_clause
|
906
961
|
# ) -> BodyStmt
|
907
962
|
def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause)
|
963
|
+
# In certain versions of Ruby, the `statements` argument can be any node
|
964
|
+
# in the case that we're inside of an endless method definition. In this
|
965
|
+
# case we'll wrap it in a Statements node to be consistent.
|
966
|
+
unless statements.is_a?(Statements)
|
967
|
+
statements =
|
968
|
+
Statements.new(body: [statements], location: statements.location)
|
969
|
+
end
|
970
|
+
|
908
971
|
parts = [statements, rescue_clause, else_clause, ensure_clause].compact
|
909
972
|
|
910
973
|
BodyStmt.new(
|
@@ -929,6 +992,7 @@ module SyntaxTree
|
|
929
992
|
|
930
993
|
start_char = find_next_statement_start(location.end_char)
|
931
994
|
statements.bind(
|
995
|
+
self,
|
932
996
|
start_char,
|
933
997
|
start_char - line_counts[location.start_line - 1].start,
|
934
998
|
rbrace.location.start_char,
|
@@ -1036,6 +1100,7 @@ module SyntaxTree
|
|
1036
1100
|
start_char = find_next_statement_start(location.end_char)
|
1037
1101
|
|
1038
1102
|
bodystmt.bind(
|
1103
|
+
self,
|
1039
1104
|
start_char,
|
1040
1105
|
start_char - line_counts[location.start_line - 1].start,
|
1041
1106
|
ending.location.start_char,
|
@@ -1154,13 +1219,23 @@ module SyntaxTree
|
|
1154
1219
|
end
|
1155
1220
|
|
1156
1221
|
# :call-seq:
|
1157
|
-
# on_const_path_field: (untyped parent, Const constant) ->
|
1222
|
+
# on_const_path_field: (untyped parent, Const constant) ->
|
1223
|
+
# ConstPathField | Field
|
1158
1224
|
def on_const_path_field(parent, constant)
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1163
|
-
|
1225
|
+
if constant.is_a?(Const)
|
1226
|
+
ConstPathField.new(
|
1227
|
+
parent: parent,
|
1228
|
+
constant: constant,
|
1229
|
+
location: parent.location.to(constant.location)
|
1230
|
+
)
|
1231
|
+
else
|
1232
|
+
Field.new(
|
1233
|
+
parent: parent,
|
1234
|
+
operator: consume_operator(:"::"),
|
1235
|
+
name: constant,
|
1236
|
+
location: parent.location.to(constant.location)
|
1237
|
+
)
|
1238
|
+
end
|
1164
1239
|
end
|
1165
1240
|
|
1166
1241
|
# :call-seq:
|
@@ -1235,6 +1310,7 @@ module SyntaxTree
|
|
1235
1310
|
start_char = find_next_statement_start(params.location.end_char)
|
1236
1311
|
|
1237
1312
|
bodystmt.bind(
|
1313
|
+
self,
|
1238
1314
|
start_char,
|
1239
1315
|
start_char - line_counts[params.location.start_line - 1].start,
|
1240
1316
|
ending.location.start_char,
|
@@ -1323,6 +1399,7 @@ module SyntaxTree
|
|
1323
1399
|
start_char = find_next_statement_start(params.location.end_char)
|
1324
1400
|
|
1325
1401
|
bodystmt.bind(
|
1402
|
+
self,
|
1326
1403
|
start_char,
|
1327
1404
|
start_char - line_counts[params.location.start_line - 1].start,
|
1328
1405
|
ending.location.start_char,
|
@@ -1362,6 +1439,7 @@ module SyntaxTree
|
|
1362
1439
|
start_char = find_next_statement_start(location.end_char)
|
1363
1440
|
|
1364
1441
|
bodystmt.bind(
|
1442
|
+
self,
|
1365
1443
|
start_char,
|
1366
1444
|
start_char - line_counts[location.start_line - 1].start,
|
1367
1445
|
ending.location.start_char,
|
@@ -1457,6 +1535,7 @@ module SyntaxTree
|
|
1457
1535
|
|
1458
1536
|
start_char = find_next_statement_start(keyword.location.end_char)
|
1459
1537
|
statements.bind(
|
1538
|
+
self,
|
1460
1539
|
start_char,
|
1461
1540
|
start_char - line_counts[keyword.location.start_line - 1].start,
|
1462
1541
|
ending.location.start_char,
|
@@ -1482,6 +1561,7 @@ module SyntaxTree
|
|
1482
1561
|
|
1483
1562
|
start_char = find_next_statement_start(predicate.location.end_char)
|
1484
1563
|
statements.bind(
|
1564
|
+
self,
|
1485
1565
|
start_char,
|
1486
1566
|
start_char - line_counts[predicate.location.start_line - 1].start,
|
1487
1567
|
ending.location.start_char,
|
@@ -1605,6 +1685,7 @@ module SyntaxTree
|
|
1605
1685
|
ending = find_keyword(:end)
|
1606
1686
|
start_char = find_next_statement_start(keyword.location.end_char)
|
1607
1687
|
statements.bind(
|
1688
|
+
self,
|
1608
1689
|
start_char,
|
1609
1690
|
start_char - line_counts[keyword.location.start_line - 1].start,
|
1610
1691
|
ending.location.start_char,
|
@@ -1679,6 +1760,22 @@ module SyntaxTree
|
|
1679
1760
|
# VarField right
|
1680
1761
|
# ) -> FndPtn
|
1681
1762
|
def on_fndptn(constant, left, values, right)
|
1763
|
+
# The left and right of a find pattern are always going to be splats, so
|
1764
|
+
# we're going to consume the * operators and use their location
|
1765
|
+
# information to extend the location of the splats.
|
1766
|
+
right, left =
|
1767
|
+
[right, left].map do |node|
|
1768
|
+
operator = consume_operator(:*)
|
1769
|
+
location =
|
1770
|
+
if node.value
|
1771
|
+
operator.location.to(node.location)
|
1772
|
+
else
|
1773
|
+
operator.location
|
1774
|
+
end
|
1775
|
+
|
1776
|
+
node.copy(location: location)
|
1777
|
+
end
|
1778
|
+
|
1682
1779
|
# The opening of this find pattern is either going to be a left bracket, a
|
1683
1780
|
# right left parenthesis, or the left splat. We're going to use this to
|
1684
1781
|
# determine how to find the closing of the pattern, as well as determining
|
@@ -1719,21 +1816,20 @@ module SyntaxTree
|
|
1719
1816
|
in_keyword = consume_keyword(:in)
|
1720
1817
|
ending = consume_keyword(:end)
|
1721
1818
|
|
1722
|
-
|
1723
|
-
|
1724
|
-
|
1725
|
-
|
1726
|
-
|
1727
|
-
keyword.location.end_char < ending.location.start_char
|
1728
|
-
tokens.delete(keyword)
|
1729
|
-
end
|
1819
|
+
delimiter =
|
1820
|
+
find_keyword_between(:do, collection, ending) ||
|
1821
|
+
find_token_between(Semicolon, collection, ending)
|
1822
|
+
|
1823
|
+
tokens.delete(delimiter) if delimiter
|
1730
1824
|
|
1731
1825
|
start_char =
|
1732
|
-
find_next_statement_start((
|
1826
|
+
find_next_statement_start((delimiter || collection).location.end_char)
|
1827
|
+
|
1733
1828
|
statements.bind(
|
1829
|
+
self,
|
1734
1830
|
start_char,
|
1735
1831
|
start_char -
|
1736
|
-
line_counts[(
|
1832
|
+
line_counts[(delimiter || collection).location.end_line - 1].start,
|
1737
1833
|
ending.location.start_char,
|
1738
1834
|
ending.location.start_column
|
1739
1835
|
)
|
@@ -1787,7 +1883,7 @@ module SyntaxTree
|
|
1787
1883
|
line: lineno,
|
1788
1884
|
char: char_pos,
|
1789
1885
|
column: current_column,
|
1790
|
-
size: value.size
|
1886
|
+
size: value.size
|
1791
1887
|
)
|
1792
1888
|
|
1793
1889
|
# Here we're going to artificially create an extra node type so that if
|
@@ -1822,7 +1918,7 @@ module SyntaxTree
|
|
1822
1918
|
line: lineno,
|
1823
1919
|
char: char_pos,
|
1824
1920
|
column: current_column,
|
1825
|
-
size: value.size
|
1921
|
+
size: value.size
|
1826
1922
|
)
|
1827
1923
|
|
1828
1924
|
heredoc_end = HeredocEnd.new(value: value.chomp, location: location)
|
@@ -1837,9 +1933,9 @@ module SyntaxTree
|
|
1837
1933
|
start_line: heredoc.location.start_line,
|
1838
1934
|
start_char: heredoc.location.start_char,
|
1839
1935
|
start_column: heredoc.location.start_column,
|
1840
|
-
end_line:
|
1841
|
-
end_char:
|
1842
|
-
end_column:
|
1936
|
+
end_line: location.end_line,
|
1937
|
+
end_char: location.end_char,
|
1938
|
+
end_column: location.end_column
|
1843
1939
|
)
|
1844
1940
|
)
|
1845
1941
|
end
|
@@ -1847,10 +1943,42 @@ module SyntaxTree
|
|
1847
1943
|
# :call-seq:
|
1848
1944
|
# on_hshptn: (
|
1849
1945
|
# (nil | untyped) constant,
|
1850
|
-
# Array[[Label, untyped]] keywords,
|
1946
|
+
# Array[[Label | StringContent, untyped]] keywords,
|
1851
1947
|
# (nil | VarField) keyword_rest
|
1852
1948
|
# ) -> HshPtn
|
1853
1949
|
def on_hshptn(constant, keywords, keyword_rest)
|
1950
|
+
keywords =
|
1951
|
+
(keywords || []).map do |(label, value)|
|
1952
|
+
if label.is_a?(Label)
|
1953
|
+
[label, value]
|
1954
|
+
else
|
1955
|
+
tstring_beg_index =
|
1956
|
+
tokens.rindex do |token|
|
1957
|
+
token.is_a?(TStringBeg) &&
|
1958
|
+
token.location.start_char < label.location.start_char
|
1959
|
+
end
|
1960
|
+
|
1961
|
+
tstring_beg = tokens.delete_at(tstring_beg_index)
|
1962
|
+
|
1963
|
+
label_end_index =
|
1964
|
+
tokens.rindex do |token|
|
1965
|
+
token.is_a?(LabelEnd) &&
|
1966
|
+
token.location.start_char == label.location.end_char
|
1967
|
+
end
|
1968
|
+
|
1969
|
+
label_end = tokens.delete_at(label_end_index)
|
1970
|
+
|
1971
|
+
[
|
1972
|
+
DynaSymbol.new(
|
1973
|
+
parts: label.parts,
|
1974
|
+
quote: label_end.value[0],
|
1975
|
+
location: tstring_beg.location.to(label_end.location)
|
1976
|
+
),
|
1977
|
+
value
|
1978
|
+
]
|
1979
|
+
end
|
1980
|
+
end
|
1981
|
+
|
1854
1982
|
if keyword_rest
|
1855
1983
|
# We're doing this to delete the token from the list so that it doesn't
|
1856
1984
|
# confuse future patterns by thinking they have an extra ** on the end.
|
@@ -1863,7 +1991,7 @@ module SyntaxTree
|
|
1863
1991
|
keyword_rest = VarField.new(value: nil, location: token.location)
|
1864
1992
|
end
|
1865
1993
|
|
1866
|
-
parts = [constant, *keywords
|
1994
|
+
parts = [constant, *keywords.flatten(1), keyword_rest].compact
|
1867
1995
|
|
1868
1996
|
# If there's no constant, there may be braces, so we're going to look for
|
1869
1997
|
# those to get our bounds.
|
@@ -1880,7 +2008,7 @@ module SyntaxTree
|
|
1880
2008
|
|
1881
2009
|
HshPtn.new(
|
1882
2010
|
constant: constant,
|
1883
|
-
keywords: keywords
|
2011
|
+
keywords: keywords,
|
1884
2012
|
keyword_rest: keyword_rest,
|
1885
2013
|
location: parts[0].location.to(parts[-1].location)
|
1886
2014
|
)
|
@@ -1911,8 +2039,14 @@ module SyntaxTree
|
|
1911
2039
|
beginning = consume_keyword(:if)
|
1912
2040
|
ending = consequent || consume_keyword(:end)
|
1913
2041
|
|
1914
|
-
|
2042
|
+
if (keyword = find_keyword_between(:then, predicate, ending))
|
2043
|
+
tokens.delete(keyword)
|
2044
|
+
end
|
2045
|
+
|
2046
|
+
start_char =
|
2047
|
+
find_next_statement_start((keyword || predicate).location.end_char)
|
1915
2048
|
statements.bind(
|
2049
|
+
self,
|
1916
2050
|
start_char,
|
1917
2051
|
start_char - line_counts[predicate.location.end_line - 1].start,
|
1918
2052
|
ending.location.start_char,
|
@@ -1946,7 +2080,7 @@ module SyntaxTree
|
|
1946
2080
|
IfNode.new(
|
1947
2081
|
predicate: predicate,
|
1948
2082
|
statements:
|
1949
|
-
Statements.new(
|
2083
|
+
Statements.new(body: [statement], location: statement.location),
|
1950
2084
|
consequent: nil,
|
1951
2085
|
location: statement.location.to(predicate.location)
|
1952
2086
|
)
|
@@ -1995,8 +2129,10 @@ module SyntaxTree
|
|
1995
2129
|
statements_start = token
|
1996
2130
|
end
|
1997
2131
|
|
1998
|
-
start_char =
|
2132
|
+
start_char =
|
2133
|
+
find_next_statement_start((token || statements_start).location.end_char)
|
1999
2134
|
statements.bind(
|
2135
|
+
self,
|
2000
2136
|
start_char,
|
2001
2137
|
start_char -
|
2002
2138
|
line_counts[statements_start.location.start_line - 1].start,
|
@@ -2121,12 +2257,19 @@ module SyntaxTree
|
|
2121
2257
|
token.location.start_char > beginning.location.start_char
|
2122
2258
|
end
|
2123
2259
|
|
2260
|
+
if braces
|
2261
|
+
opening = consume_token(TLamBeg)
|
2262
|
+
closing = consume_token(RBrace)
|
2263
|
+
else
|
2264
|
+
opening = consume_keyword(:do)
|
2265
|
+
closing = consume_keyword(:end)
|
2266
|
+
end
|
2267
|
+
|
2124
2268
|
# We need to do some special mapping here. Since ripper doesn't support
|
2125
|
-
# capturing lambda
|
2269
|
+
# capturing lambda vars, we need to normalize all of that here.
|
2126
2270
|
params =
|
2127
|
-
|
2128
|
-
|
2129
|
-
# In this case we've gotten to the <3.2 parentheses wrapping a set of
|
2271
|
+
if params.is_a?(Paren)
|
2272
|
+
# In this case we've gotten to the parentheses wrapping a set of
|
2130
2273
|
# parameters case. Here we need to manually scan for lambda locals.
|
2131
2274
|
range = (params.location.start_char + 1)...params.location.end_char
|
2132
2275
|
locals = lambda_locals(source[range])
|
@@ -2148,27 +2291,31 @@ module SyntaxTree
|
|
2148
2291
|
|
2149
2292
|
node.comments.concat(params.comments)
|
2150
2293
|
node
|
2151
|
-
|
2152
|
-
#
|
2153
|
-
#
|
2154
|
-
#
|
2294
|
+
else
|
2295
|
+
# If there are no parameters, then we didn't have anything to base the
|
2296
|
+
# location information of off. Now that we have an opening of the
|
2297
|
+
# block, we can correct this.
|
2298
|
+
if params.empty?
|
2299
|
+
opening_location = opening.location
|
2300
|
+
location =
|
2301
|
+
Location.fixed(
|
2302
|
+
line: opening_location.start_line,
|
2303
|
+
char: opening_location.start_char,
|
2304
|
+
column: opening_location.start_column
|
2305
|
+
)
|
2306
|
+
|
2307
|
+
params = params.copy(location: location)
|
2308
|
+
end
|
2309
|
+
|
2310
|
+
# In this case we've gotten to the plain set of parameters. In this
|
2311
|
+
# case there cannot be lambda locals, so we will wrap the parameters
|
2312
|
+
# into a lambda var that has no locals.
|
2155
2313
|
LambdaVar.new(params: params, locals: [], location: params.location)
|
2156
|
-
when LambdaVar
|
2157
|
-
# In this case we've gotten to 3.2+ lambda var. In this case we don't
|
2158
|
-
# need to do anything and can just the value as given.
|
2159
|
-
params
|
2160
2314
|
end
|
2161
2315
|
|
2162
|
-
if braces
|
2163
|
-
opening = consume_token(TLamBeg)
|
2164
|
-
closing = consume_token(RBrace)
|
2165
|
-
else
|
2166
|
-
opening = consume_keyword(:do)
|
2167
|
-
closing = consume_keyword(:end)
|
2168
|
-
end
|
2169
|
-
|
2170
2316
|
start_char = find_next_statement_start(opening.location.end_char)
|
2171
2317
|
statements.bind(
|
2318
|
+
self,
|
2172
2319
|
start_char,
|
2173
2320
|
start_char - line_counts[opening.location.end_line - 1].start,
|
2174
2321
|
closing.location.start_char,
|
@@ -2353,23 +2500,30 @@ module SyntaxTree
|
|
2353
2500
|
|
2354
2501
|
# :call-seq:
|
2355
2502
|
# on_method_add_block: (
|
2356
|
-
# (Call | Command | CommandCall) call,
|
2503
|
+
# (Break | Call | Command | CommandCall, Next) call,
|
2357
2504
|
# Block block
|
2358
|
-
# ) -> MethodAddBlock
|
2505
|
+
# ) -> Break | MethodAddBlock
|
2359
2506
|
def on_method_add_block(call, block)
|
2360
2507
|
location = call.location.to(block.location)
|
2361
2508
|
|
2362
2509
|
case call
|
2510
|
+
when Break, Next, ReturnNode
|
2511
|
+
parts = call.arguments.parts
|
2512
|
+
|
2513
|
+
node = parts.pop
|
2514
|
+
copied =
|
2515
|
+
node.copy(block: block, location: node.location.to(block.location))
|
2516
|
+
|
2517
|
+
copied.comments.concat(call.comments)
|
2518
|
+
parts << copied
|
2519
|
+
|
2520
|
+
call.copy(location: location)
|
2363
2521
|
when Command, CommandCall
|
2364
2522
|
node = call.copy(block: block, location: location)
|
2365
2523
|
node.comments.concat(call.comments)
|
2366
2524
|
node
|
2367
2525
|
else
|
2368
|
-
MethodAddBlock.new(
|
2369
|
-
call: call,
|
2370
|
-
block: block,
|
2371
|
-
location: call.location.to(block.location)
|
2372
|
-
)
|
2526
|
+
MethodAddBlock.new(call: call, block: block, location: location)
|
2373
2527
|
end
|
2374
2528
|
end
|
2375
2529
|
|
@@ -2446,6 +2600,7 @@ module SyntaxTree
|
|
2446
2600
|
start_char = find_next_statement_start(constant.location.end_char)
|
2447
2601
|
|
2448
2602
|
bodystmt.bind(
|
2603
|
+
self,
|
2449
2604
|
start_char,
|
2450
2605
|
start_char - line_counts[constant.location.start_line - 1].start,
|
2451
2606
|
ending.location.start_char,
|
@@ -2592,19 +2747,40 @@ module SyntaxTree
|
|
2592
2747
|
# have a `nil` for the value instead of a `false`.
|
2593
2748
|
keywords&.map! { |(key, value)| [key, value || nil] }
|
2594
2749
|
|
2595
|
-
|
2596
|
-
|
2597
|
-
|
2598
|
-
|
2599
|
-
|
2600
|
-
|
2601
|
-
|
2602
|
-
|
2603
|
-
|
2750
|
+
# Here we're going to build up a list of all of the params so that we can
|
2751
|
+
# determine our location information.
|
2752
|
+
parts = []
|
2753
|
+
|
2754
|
+
requireds&.each { |required| parts << required.location }
|
2755
|
+
optionals&.each do |(key, value)|
|
2756
|
+
parts << key.location
|
2757
|
+
parts << value.location if value
|
2758
|
+
end
|
2759
|
+
|
2760
|
+
parts << rest.location if rest
|
2761
|
+
posts&.each { |post| parts << post.location }
|
2762
|
+
|
2763
|
+
keywords&.each do |(key, value)|
|
2764
|
+
parts << key.location
|
2765
|
+
parts << value.location if value
|
2766
|
+
end
|
2767
|
+
|
2768
|
+
if keyword_rest == :nil
|
2769
|
+
# When we get a :nil here, it means that we have **nil syntax, which
|
2770
|
+
# means this set of parameters accepts no more keyword arguments. In
|
2771
|
+
# this case we need to go and find the location of these two tokens.
|
2772
|
+
operator = consume_operator(:**)
|
2773
|
+
parts << operator.location.to(consume_keyword(:nil).location)
|
2774
|
+
elsif keyword_rest
|
2775
|
+
parts << keyword_rest.location
|
2776
|
+
end
|
2777
|
+
|
2778
|
+
parts << block.location if block && block != :&
|
2779
|
+
parts = parts.compact
|
2604
2780
|
|
2605
2781
|
location =
|
2606
2782
|
if parts.any?
|
2607
|
-
parts[0].
|
2783
|
+
parts[0].to(parts[-1])
|
2608
2784
|
else
|
2609
2785
|
Location.fixed(line: lineno, char: char_pos, column: current_column)
|
2610
2786
|
end
|
@@ -2701,7 +2877,7 @@ module SyntaxTree
|
|
2701
2877
|
)
|
2702
2878
|
|
2703
2879
|
statements.body << @__end__ if @__end__
|
2704
|
-
statements.bind(0, 0, source.length, last_column)
|
2880
|
+
statements.bind(self, 0, 0, source.length, last_column)
|
2705
2881
|
|
2706
2882
|
program = Program.new(statements: statements, location: location)
|
2707
2883
|
attach_comments(program, @comments)
|
@@ -3033,8 +3209,9 @@ module SyntaxTree
|
|
3033
3209
|
exceptions = exceptions[0] if exceptions.is_a?(Array)
|
3034
3210
|
|
3035
3211
|
last_node = variable || exceptions || keyword
|
3036
|
-
start_char = find_next_statement_start(last_node.
|
3212
|
+
start_char = find_next_statement_start(last_node.end_char)
|
3037
3213
|
statements.bind(
|
3214
|
+
self,
|
3038
3215
|
start_char,
|
3039
3216
|
start_char - line_counts[last_node.location.start_line - 1].start,
|
3040
3217
|
char_pos,
|
@@ -3055,7 +3232,7 @@ module SyntaxTree
|
|
3055
3232
|
start_char: keyword.location.end_char + 1,
|
3056
3233
|
start_column: keyword.location.end_column + 1,
|
3057
3234
|
end_line: last_node.location.end_line,
|
3058
|
-
end_char: last_node.
|
3235
|
+
end_char: last_node.end_char,
|
3059
3236
|
end_column: last_node.location.end_column
|
3060
3237
|
)
|
3061
3238
|
)
|
@@ -3153,6 +3330,7 @@ module SyntaxTree
|
|
3153
3330
|
start_char = find_next_statement_start(target.location.end_char)
|
3154
3331
|
|
3155
3332
|
bodystmt.bind(
|
3333
|
+
self,
|
3156
3334
|
start_char,
|
3157
3335
|
start_char - line_counts[target.location.start_line - 1].start,
|
3158
3336
|
ending.location.start_char,
|
@@ -3166,9 +3344,29 @@ module SyntaxTree
|
|
3166
3344
|
)
|
3167
3345
|
end
|
3168
3346
|
|
3169
|
-
#
|
3170
|
-
#
|
3171
|
-
#
|
3347
|
+
# Semicolons are tokens that get added to the token list but never get
|
3348
|
+
# attached to the AST. Because of this they only need to track their
|
3349
|
+
# associated location so they can be used for computing bounds.
|
3350
|
+
class Semicolon
|
3351
|
+
attr_reader :location
|
3352
|
+
|
3353
|
+
def initialize(location)
|
3354
|
+
@location = location
|
3355
|
+
end
|
3356
|
+
end
|
3357
|
+
|
3358
|
+
# :call-seq:
|
3359
|
+
# on_semicolon: (String value) -> Semicolon
|
3360
|
+
def on_semicolon(value)
|
3361
|
+
tokens << Semicolon.new(
|
3362
|
+
Location.token(
|
3363
|
+
line: lineno,
|
3364
|
+
char: char_pos,
|
3365
|
+
column: current_column,
|
3366
|
+
size: value.size
|
3367
|
+
)
|
3368
|
+
)
|
3369
|
+
end
|
3172
3370
|
|
3173
3371
|
# def on_sp(value)
|
3174
3372
|
# value
|
@@ -3186,18 +3384,13 @@ module SyntaxTree
|
|
3186
3384
|
statements.location.to(statement.location)
|
3187
3385
|
end
|
3188
3386
|
|
3189
|
-
Statements.new(
|
3190
|
-
self,
|
3191
|
-
body: statements.body << statement,
|
3192
|
-
location: location
|
3193
|
-
)
|
3387
|
+
Statements.new(body: statements.body << statement, location: location)
|
3194
3388
|
end
|
3195
3389
|
|
3196
3390
|
# :call-seq:
|
3197
3391
|
# on_stmts_new: () -> Statements
|
3198
3392
|
def on_stmts_new
|
3199
3393
|
Statements.new(
|
3200
|
-
self,
|
3201
3394
|
body: [],
|
3202
3395
|
location:
|
3203
3396
|
Location.fixed(line: lineno, char: char_pos, column: current_column)
|
@@ -3262,6 +3455,7 @@ module SyntaxTree
|
|
3262
3455
|
embexpr_end = consume_token(EmbExprEnd)
|
3263
3456
|
|
3264
3457
|
statements.bind(
|
3458
|
+
self,
|
3265
3459
|
embexpr_beg.location.end_char,
|
3266
3460
|
embexpr_beg.location.end_column,
|
3267
3461
|
embexpr_end.location.start_char,
|
@@ -3605,8 +3799,14 @@ module SyntaxTree
|
|
3605
3799
|
beginning = consume_keyword(:unless)
|
3606
3800
|
ending = consequent || consume_keyword(:end)
|
3607
3801
|
|
3608
|
-
|
3802
|
+
if (keyword = find_keyword_between(:then, predicate, ending))
|
3803
|
+
tokens.delete(keyword)
|
3804
|
+
end
|
3805
|
+
|
3806
|
+
start_char =
|
3807
|
+
find_next_statement_start((keyword || predicate).location.end_char)
|
3609
3808
|
statements.bind(
|
3809
|
+
self,
|
3610
3810
|
start_char,
|
3611
3811
|
start_char - line_counts[predicate.location.end_line - 1].start,
|
3612
3812
|
ending.location.start_char,
|
@@ -3629,7 +3829,7 @@ module SyntaxTree
|
|
3629
3829
|
UnlessNode.new(
|
3630
3830
|
predicate: predicate,
|
3631
3831
|
statements:
|
3632
|
-
Statements.new(
|
3832
|
+
Statements.new(body: [statement], location: statement.location),
|
3633
3833
|
consequent: nil,
|
3634
3834
|
location: statement.location.to(predicate.location)
|
3635
3835
|
)
|
@@ -3641,17 +3841,18 @@ module SyntaxTree
|
|
3641
3841
|
beginning = consume_keyword(:until)
|
3642
3842
|
ending = consume_keyword(:end)
|
3643
3843
|
|
3644
|
-
|
3645
|
-
|
3646
|
-
|
3647
|
-
|
3648
|
-
|
3649
|
-
tokens.delete(keyword)
|
3650
|
-
end
|
3844
|
+
delimiter =
|
3845
|
+
find_keyword_between(:do, predicate, statements) ||
|
3846
|
+
find_token_between(Semicolon, predicate, statements)
|
3847
|
+
|
3848
|
+
tokens.delete(delimiter) if delimiter
|
3651
3849
|
|
3652
3850
|
# Update the Statements location information
|
3653
|
-
start_char =
|
3851
|
+
start_char =
|
3852
|
+
find_next_statement_start((delimiter || predicate).location.end_char)
|
3853
|
+
|
3654
3854
|
statements.bind(
|
3855
|
+
self,
|
3655
3856
|
start_char,
|
3656
3857
|
start_char - line_counts[predicate.location.end_line - 1].start,
|
3657
3858
|
ending.location.start_char,
|
@@ -3673,7 +3874,7 @@ module SyntaxTree
|
|
3673
3874
|
UntilNode.new(
|
3674
3875
|
predicate: predicate,
|
3675
3876
|
statements:
|
3676
|
-
Statements.new(
|
3877
|
+
Statements.new(body: [statement], location: statement.location),
|
3677
3878
|
location: statement.location.to(predicate.location)
|
3678
3879
|
)
|
3679
3880
|
end
|
@@ -3744,9 +3945,11 @@ module SyntaxTree
|
|
3744
3945
|
statements_start = token
|
3745
3946
|
end
|
3746
3947
|
|
3747
|
-
start_char =
|
3948
|
+
start_char =
|
3949
|
+
find_next_statement_start((token || statements_start).location.end_char)
|
3748
3950
|
|
3749
3951
|
statements.bind(
|
3952
|
+
self,
|
3750
3953
|
start_char,
|
3751
3954
|
start_char -
|
3752
3955
|
line_counts[statements_start.location.start_line - 1].start,
|
@@ -3768,17 +3971,18 @@ module SyntaxTree
|
|
3768
3971
|
beginning = consume_keyword(:while)
|
3769
3972
|
ending = consume_keyword(:end)
|
3770
3973
|
|
3771
|
-
|
3772
|
-
|
3773
|
-
|
3774
|
-
|
3775
|
-
|
3776
|
-
tokens.delete(keyword)
|
3777
|
-
end
|
3974
|
+
delimiter =
|
3975
|
+
find_keyword_between(:do, predicate, statements) ||
|
3976
|
+
find_token_between(Semicolon, predicate, statements)
|
3977
|
+
|
3978
|
+
tokens.delete(delimiter) if delimiter
|
3778
3979
|
|
3779
3980
|
# Update the Statements location information
|
3780
|
-
start_char =
|
3981
|
+
start_char =
|
3982
|
+
find_next_statement_start((delimiter || predicate).location.end_char)
|
3983
|
+
|
3781
3984
|
statements.bind(
|
3985
|
+
self,
|
3782
3986
|
start_char,
|
3783
3987
|
start_char - line_counts[predicate.location.end_line - 1].start,
|
3784
3988
|
ending.location.start_char,
|
@@ -3800,7 +4004,7 @@ module SyntaxTree
|
|
3800
4004
|
WhileNode.new(
|
3801
4005
|
predicate: predicate,
|
3802
4006
|
statements:
|
3803
|
-
Statements.new(
|
4007
|
+
Statements.new(body: [statement], location: statement.location),
|
3804
4008
|
location: statement.location.to(predicate.location)
|
3805
4009
|
)
|
3806
4010
|
end
|