syntax_tree 5.3.0 → 6.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -1
- data/CHANGELOG.md +78 -1
- data/Gemfile.lock +7 -7
- data/README.md +33 -9
- data/Rakefile +12 -8
- data/bin/console +1 -0
- data/bin/whitequark +79 -0
- data/doc/changing_structure.md +16 -0
- data/lib/syntax_tree/basic_visitor.rb +44 -5
- data/lib/syntax_tree/cli.rb +2 -2
- data/lib/syntax_tree/dsl.rb +23 -11
- data/lib/syntax_tree/{visitor/field_visitor.rb → field_visitor.rb} +54 -55
- data/lib/syntax_tree/formatter.rb +1 -1
- data/lib/syntax_tree/index.rb +158 -59
- data/lib/syntax_tree/json_visitor.rb +55 -0
- data/lib/syntax_tree/language_server.rb +157 -2
- data/lib/syntax_tree/match_visitor.rb +120 -0
- data/lib/syntax_tree/mermaid.rb +177 -0
- data/lib/syntax_tree/mermaid_visitor.rb +69 -0
- data/lib/syntax_tree/{visitor/mutation_visitor.rb → mutation_visitor.rb} +27 -27
- data/lib/syntax_tree/node.rb +245 -123
- data/lib/syntax_tree/parser.rb +332 -119
- data/lib/syntax_tree/pretty_print_visitor.rb +83 -0
- data/lib/syntax_tree/reflection.rb +241 -0
- data/lib/syntax_tree/translation/parser.rb +3107 -0
- data/lib/syntax_tree/translation/rubocop_ast.rb +21 -0
- data/lib/syntax_tree/translation.rb +28 -0
- data/lib/syntax_tree/version.rb +1 -1
- data/lib/syntax_tree/with_scope.rb +244 -0
- data/lib/syntax_tree/yarv/basic_block.rb +53 -0
- data/lib/syntax_tree/yarv/calldata.rb +91 -0
- data/lib/syntax_tree/yarv/compiler.rb +110 -100
- data/lib/syntax_tree/yarv/control_flow_graph.rb +257 -0
- data/lib/syntax_tree/yarv/data_flow_graph.rb +338 -0
- data/lib/syntax_tree/yarv/decompiler.rb +1 -1
- data/lib/syntax_tree/yarv/disassembler.rb +104 -80
- data/lib/syntax_tree/yarv/instruction_sequence.rb +43 -18
- data/lib/syntax_tree/yarv/instructions.rb +203 -649
- data/lib/syntax_tree/yarv/legacy.rb +12 -24
- data/lib/syntax_tree/yarv/sea_of_nodes.rb +534 -0
- data/lib/syntax_tree/yarv.rb +18 -0
- data/lib/syntax_tree.rb +88 -56
- data/tasks/sorbet.rake +277 -0
- data/tasks/whitequark.rake +87 -0
- metadata +23 -11
- data/.gitmodules +0 -9
- data/lib/syntax_tree/language_server/inlay_hints.rb +0 -159
- data/lib/syntax_tree/visitor/environment.rb +0 -84
- data/lib/syntax_tree/visitor/json_visitor.rb +0 -55
- data/lib/syntax_tree/visitor/match_visitor.rb +0 -122
- data/lib/syntax_tree/visitor/pretty_print_visitor.rb +0 -85
- data/lib/syntax_tree/visitor/with_environment.rb +0 -140
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b1e39c546fb96aea864e8190b8a8ff338543afd644d2334edad5b98663d418f2
|
4
|
+
data.tar.gz: 7633b68e7b42fcd782fc1c89395d93679e72828fe1a9214e741ad00718543c6b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a0990a769cbc448e549d14542610ccfbf09536ada139cc10701871a506a1b160a8ce751712a94a8f003d3b2c1828a6b5d4bd5384efe63f0f82a21c0762b689d5
|
7
|
+
data.tar.gz: a08b4057370c230bfd713ff9a717ea4f2a02be4f8610af51b18143ff9b2b9e542541b77c049488c500033751b50adc279d5e3a8beb0568f5e2bb165793c612d4
|
data/.rubocop.yml
CHANGED
@@ -8,7 +8,6 @@ AllCops:
|
|
8
8
|
TargetRubyVersion: 2.7
|
9
9
|
Exclude:
|
10
10
|
- '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*'
|
11
|
-
- test/ruby-syntax-fixtures/**/*
|
12
11
|
- test.rb
|
13
12
|
|
14
13
|
Gemspec/DevelopmentDependencies:
|
@@ -29,6 +28,9 @@ Lint/AmbiguousRange:
|
|
29
28
|
Lint/BooleanSymbol:
|
30
29
|
Enabled: false
|
31
30
|
|
31
|
+
Lint/Debugger:
|
32
|
+
Enabled: false
|
33
|
+
|
32
34
|
Lint/DuplicateBranch:
|
33
35
|
Enabled: false
|
34
36
|
|
@@ -80,6 +82,9 @@ Security/Eval:
|
|
80
82
|
Style/AccessorGrouping:
|
81
83
|
Enabled: false
|
82
84
|
|
85
|
+
Style/Alias:
|
86
|
+
Enabled: false
|
87
|
+
|
83
88
|
Style/CaseEquality:
|
84
89
|
Enabled: false
|
85
90
|
|
@@ -89,6 +94,9 @@ Style/CaseLikeIf:
|
|
89
94
|
Style/ClassVars:
|
90
95
|
Enabled: false
|
91
96
|
|
97
|
+
Style/CombinableLoops:
|
98
|
+
Enabled: false
|
99
|
+
|
92
100
|
Style/DocumentDynamicEvalDefinition:
|
93
101
|
Enabled: false
|
94
102
|
|
@@ -110,6 +118,9 @@ Style/FormatStringToken:
|
|
110
118
|
Style/GuardClause:
|
111
119
|
Enabled: false
|
112
120
|
|
121
|
+
Style/HashLikeCase:
|
122
|
+
Enabled: false
|
123
|
+
|
113
124
|
Style/IdenticalConditionalBranches:
|
114
125
|
Enabled: false
|
115
126
|
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,81 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [6.0.1] - 2023-02-26
|
10
|
+
|
11
|
+
### Added
|
12
|
+
|
13
|
+
- The class declarations returned as the result of the indexing operation now have their superclass as a field. It is returned as an array of constants. If the superclass is anything other than a constant lookup, then it raises an error.
|
14
|
+
|
15
|
+
### Changed
|
16
|
+
|
17
|
+
- The `nesting` field on the results of the indexing operation is no longer a single flat array. Instead it is an array of arrays, where each array is a single nesting level. This more accurately reflects the nesting of the nodes in the tree. For example, `class Foo::Bar::Baz; end` would result in `[Foo, Bar, Baz]`, but that incorrectly implies that you can see constants at each of those levels. Now this would result in `[[Foo, Bar, Baz]]` to indicate that it can see either the top level or constants within the scope of `Foo::Bar::Baz` only.
|
18
|
+
- When formatting hashes that have omitted values and mixed hash rockets with labels, the formatting now maintains whichever delimiter was used in the source. This is because forcing the use of hash rockets with omitted values results in a syntax error.
|
19
|
+
- Handle the case where a bare hash is used after the `break`, `next`, or `return` keywords. Previously this would result in hash labels which is not valid syntax. Now it maintains the delimiters used in the source.
|
20
|
+
- The `<<` operator will now break on chained `<<` expressions. Previously it would always stay flat.
|
21
|
+
|
22
|
+
## [6.0.0] - 2023-02-10
|
23
|
+
|
24
|
+
### Added
|
25
|
+
|
26
|
+
- `SyntaxTree::BasicVisitor::visit_methods` has been added to allow you to check multiple visit methods inside of a block. There _was_ a method called `visit_methods` previously, but it was undocumented because it was meant as a private API. That method has been renamed to `valid_visit_methods`.
|
27
|
+
- `rake sorbet:rbi` has been added as a task within the repository to generate an RBI file corresponding to the nodes in the tree. This can be used to help aid consumers of Syntax Tree that are using Sorbet.
|
28
|
+
- `SyntaxTree::Reflection` has been added to allow you to get information about the nodes in the tree. It is not required by default, since it takes a small amount of time to parse `node.rb` and get all of the information.
|
29
|
+
- `SyntaxTree::Node#to_mermaid` has been added to allow you to generate a Mermaid diagram of the node and its children. This is useful for debugging and understanding the structure of the tree.
|
30
|
+
- `SyntaxTree::Translation` has been added as an experimental API to transform the Syntax Tree syntax tree into the syntax trees represented by the whitequark/parser and rubocop/rubocop-ast gems.
|
31
|
+
- `SyntaxTree::Translation.to_parser(node, buffer)` will return a `Parser::AST::Node` object.
|
32
|
+
- `SyntaxTree::Translation.to_rubocop_ast(node, buffer)` will return a `RuboCop::AST::Node` object.
|
33
|
+
- `SyntaxTree::index` and `SyntaxTree::index_file` have been added to allow you to get a list of all of the classes, modules, and methods defined in a given source string or file.
|
34
|
+
- Various convenience methods have been added:
|
35
|
+
- `SyntaxTree::format_file` - which calls format with the result of reading the file
|
36
|
+
- `SyntaxTree::format_node` - which formats the node directly
|
37
|
+
- `SyntaxTree::parse_file` - which calls parse with the result of reading the file
|
38
|
+
- `SyntaxTree::search_file` - which calls search with the result of reading the file
|
39
|
+
- `SyntaxTree::Node#start_char` - which is the same as calling `node.location.start_char`
|
40
|
+
- `SyntaxTree::Node#end_char` - which is the same as calling `node.location.end_char`
|
41
|
+
- `SyntaxTree::Assoc` nodes can now be formatted on their own without a parent hash node.
|
42
|
+
- `SyntaxTree::BlockVar#arg0?` has been added to check if a single required block parameter is present and would potentially be expanded.
|
43
|
+
- More experimental APIs have been added to the `SyntaxTree::YARV` module, including:
|
44
|
+
- `SyntaxTree::YARV::ControlFlowGraph`
|
45
|
+
- `SyntaxTree::YARV::DataFlowGraph`
|
46
|
+
- `SyntaxTree::YARV::SeaOfNodes`
|
47
|
+
|
48
|
+
### Changed
|
49
|
+
|
50
|
+
#### Major changes
|
51
|
+
|
52
|
+
- *BREAKING* Updates to `WithEnvironment`:
|
53
|
+
- The `WithEnvironment` module has been renamed to `WithScope`.
|
54
|
+
- The `current_environment` method has been renamed to `current_scope`.
|
55
|
+
- The `with_current_environment` method has been removed.
|
56
|
+
- Previously scopes were always able to look up the tree, as in: `a = 1; def foo; a = 2; end` would see only a single `a` variable. That has been corrected.
|
57
|
+
- Previously accessing variables from inside of blocks that were not shadowed would mark them as being local to the block only. This has been correct.
|
58
|
+
- *BREAKING* Lots of constants moved out of `SyntaxTree::Visitor` to just `SyntaxTree`:
|
59
|
+
* `SyntaxTree::Visitor::FieldVisitor` is now `SyntaxTree::FieldVisitor`
|
60
|
+
* `SyntaxTree::Visitor::JSONVisitor` is now `SyntaxTree::JSONVisitor`
|
61
|
+
* `SyntaxTree::Visitor::MatchVisitor` is now `SyntaxTree::MatchVisitor`
|
62
|
+
* `SyntaxTree::Visitor::MutationVisitor` is now `SyntaxTree::MutationVisitor`
|
63
|
+
* `SyntaxTree::Visitor::PrettyPrintVisitor` is now `SyntaxTree::PrettyPrintVisitor`
|
64
|
+
- *BREAKING* Lots of constants are now autoloaded instead of required by default. This is only particularly relevant if you are in a forking environment and want to preload constants before forking for better memory usage with copy-on-write.
|
65
|
+
- *BREAKING* The `SyntaxTree::Statements#initialize` method no longer accepts a parser as the first argument. It now mirrors the other nodes in that it accepts its children and location. As a result, Syntax Tree nodes are now marshalable (and therefore can be sent over DRb). Previously the `Statements` node was not able to be marshaled because it held a reference to the parser.
|
66
|
+
|
67
|
+
#### Minor changes
|
68
|
+
|
69
|
+
- Many places where embedded documents (`=begin` to `=end`) were being treated as real comments have been fixed for formatting.
|
70
|
+
- Dynamic symbols in keyword pattern matching now have better formatting.
|
71
|
+
- Endless method definitions used to have a `SyntaxTree::BodyStmt` node that had any kind of node as its `statements` field. That has been corrected to be more consistent such that now going from `def_node.bodystmt.statements` always returns a `SyntaxTree::Statements` node, which is more consistent.
|
72
|
+
- We no longer assume that `fiddle` is able to be required, and only require it when it is actually needed.
|
73
|
+
|
74
|
+
#### Tiny changes
|
75
|
+
|
76
|
+
- Empty parameter nodes within blocks now have more accurate location information.
|
77
|
+
- Pinned variables have more correct location information now. (Previously the location was just around the variable itself, but it now includes the pin.)
|
78
|
+
- Array patterns in pattern matching now have more accurate location information when they are using parentheses with a constant present.
|
79
|
+
- Find patterns in pattern matching now have more correct location information for their `left` and `right` fields.
|
80
|
+
- Lots of nodes have more correct types in the comments on their attributes.
|
81
|
+
- The expressions `break foo.bar :baz do |qux| qux end` and `next fun foo do end` now correctly parses as a control-flow statement with a method call that has a block attached, as opposed to a control-flow statement with a block attached.
|
82
|
+
- The expression `self::a, b = 1, 2` would previously yield a `SyntaxTree::ConstPathField` node for the first element of the left-hand-side of the multiple assignment. Semantically this is incorrect, and we have fixed this to now be a `SyntaxTree::Field` node instead.
|
83
|
+
|
9
84
|
## [5.3.0] - 2023-01-26
|
10
85
|
|
11
86
|
### Added
|
@@ -497,7 +572,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
|
|
497
572
|
|
498
573
|
- 🎉 Initial release! 🎉
|
499
574
|
|
500
|
-
[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/
|
575
|
+
[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...HEAD
|
576
|
+
[6.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...v6.0.1
|
577
|
+
[6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0
|
501
578
|
[5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0
|
502
579
|
[5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0
|
503
580
|
[5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
syntax_tree (
|
4
|
+
syntax_tree (6.0.1)
|
5
5
|
prettier_print (>= 1.2.0)
|
6
6
|
|
7
7
|
GEM
|
@@ -12,25 +12,25 @@ GEM
|
|
12
12
|
json (2.6.3)
|
13
13
|
minitest (5.17.0)
|
14
14
|
parallel (1.22.1)
|
15
|
-
parser (3.2.
|
15
|
+
parser (3.2.1.0)
|
16
16
|
ast (~> 2.4.1)
|
17
17
|
prettier_print (1.2.0)
|
18
18
|
rainbow (3.1.1)
|
19
19
|
rake (13.0.6)
|
20
|
-
regexp_parser (2.
|
20
|
+
regexp_parser (2.7.0)
|
21
21
|
rexml (3.2.5)
|
22
|
-
rubocop (1.
|
22
|
+
rubocop (1.46.0)
|
23
23
|
json (~> 2.3)
|
24
24
|
parallel (~> 1.10)
|
25
25
|
parser (>= 3.2.0.0)
|
26
26
|
rainbow (>= 2.2.2, < 4.0)
|
27
27
|
regexp_parser (>= 1.8, < 3.0)
|
28
28
|
rexml (>= 3.2.5, < 4.0)
|
29
|
-
rubocop-ast (>= 1.
|
29
|
+
rubocop-ast (>= 1.26.0, < 2.0)
|
30
30
|
ruby-progressbar (~> 1.7)
|
31
31
|
unicode-display_width (>= 2.4.0, < 3.0)
|
32
|
-
rubocop-ast (1.
|
33
|
-
parser (>= 3.
|
32
|
+
rubocop-ast (1.26.0)
|
33
|
+
parser (>= 3.2.1.0)
|
34
34
|
ruby-progressbar (1.11.0)
|
35
35
|
simplecov (0.22.0)
|
36
36
|
docile (~> 1.1)
|
data/README.md
CHANGED
@@ -29,6 +29,7 @@ It is built with only standard library dependencies. It additionally ships with
|
|
29
29
|
- [SyntaxTree.format(source)](#syntaxtreeformatsource)
|
30
30
|
- [SyntaxTree.mutation(&block)](#syntaxtreemutationblock)
|
31
31
|
- [SyntaxTree.search(source, query, &block)](#syntaxtreesearchsource-query-block)
|
32
|
+
- [SyntaxTree.index(source)](#syntaxtreeindexsource)
|
32
33
|
- [Nodes](#nodes)
|
33
34
|
- [child_nodes](#child_nodes)
|
34
35
|
- [copy(**attrs)](#copyattrs)
|
@@ -40,9 +41,10 @@ It is built with only standard library dependencies. It additionally ships with
|
|
40
41
|
- [construct_keys](#construct_keys)
|
41
42
|
- [Visitor](#visitor)
|
42
43
|
- [visit_method](#visit_method)
|
44
|
+
- [visit_methods](#visit_methods)
|
43
45
|
- [BasicVisitor](#basicvisitor)
|
44
46
|
- [MutationVisitor](#mutationvisitor)
|
45
|
-
- [
|
47
|
+
- [WithScope](#withscope)
|
46
48
|
- [Language server](#language-server)
|
47
49
|
- [textDocument/formatting](#textdocumentformatting)
|
48
50
|
- [textDocument/inlayHint](#textdocumentinlayhint)
|
@@ -340,12 +342,16 @@ This function takes an input string containing Ruby code, parses it into its und
|
|
340
342
|
|
341
343
|
### SyntaxTree.mutation(&block)
|
342
344
|
|
343
|
-
This function yields a new mutation visitor to the block, and then returns the initialized visitor. It's effectively a shortcut for creating a `SyntaxTree::
|
345
|
+
This function yields a new mutation visitor to the block, and then returns the initialized visitor. It's effectively a shortcut for creating a `SyntaxTree::MutationVisitor` without having to remember the class name. For more information on that visitor, see the definition below.
|
344
346
|
|
345
347
|
### SyntaxTree.search(source, query, &block)
|
346
348
|
|
347
349
|
This function takes an input string containing Ruby code, an input string containing a valid Ruby `in` clause expression that can be used to match against nodes in the tree (can be generated using `stree expr`, `stree match`, or `Node#construct_keys`), and a block. Each node that matches the given query will be yielded to the block. The block will receive the node as its only argument.
|
348
350
|
|
351
|
+
### SyntaxTree.index(source)
|
352
|
+
|
353
|
+
This function takes an input string containing Ruby code and returns a list of all of the class declarations, module declarations, and method definitions within a file. Each of the entries also has access to its associated comments. This is useful for generating documentation or index information for a file to support something like go-to-definition.
|
354
|
+
|
349
355
|
## Nodes
|
350
356
|
|
351
357
|
There are many different node types in the syntax tree. They are meant to be treated as immutable structs containing links to child nodes with minimal logic contained within their implementation. However, for the most part they all respond to a certain set of APIs, listed below.
|
@@ -517,6 +523,26 @@ Did you mean? visit_binary
|
|
517
523
|
from bin/console:8:in `<main>'
|
518
524
|
```
|
519
525
|
|
526
|
+
### visit_methods
|
527
|
+
|
528
|
+
Similar to `visit_method`, `visit_methods` also checks that methods defined are valid visit methods. This variation however accepts a block and checks that all methods defined within that block are valid visit methods. It's meant to be used like:
|
529
|
+
|
530
|
+
```ruby
|
531
|
+
class ArithmeticVisitor < SyntaxTree::Visitor
|
532
|
+
visit_methods do
|
533
|
+
def visit_binary(node)
|
534
|
+
# ...
|
535
|
+
end
|
536
|
+
|
537
|
+
def visit_int(node)
|
538
|
+
# ...
|
539
|
+
end
|
540
|
+
end
|
541
|
+
end
|
542
|
+
```
|
543
|
+
|
544
|
+
This is only checked when the methods are defined and does not impose any kind of runtime overhead after that. It is very useful for upgrading versions of Syntax Tree in case these methods names change.
|
545
|
+
|
520
546
|
### BasicVisitor
|
521
547
|
|
522
548
|
When you're defining your own visitor, by default it will walk down the tree even if you don't define `visit_*` methods. This is to ensure you can define a subset of the necessary methods in order to only interact with the nodes you're interested in. If you'd like to change this default to instead raise an error if you visit a node you haven't explicitly handled, you can instead inherit from `BasicVisitor`.
|
@@ -537,7 +563,7 @@ The `MutationVisitor` is a visitor that can be used to mutate the tree. It works
|
|
537
563
|
|
538
564
|
```ruby
|
539
565
|
# Create a new visitor
|
540
|
-
visitor = SyntaxTree::
|
566
|
+
visitor = SyntaxTree::MutationVisitor.new
|
541
567
|
|
542
568
|
# Specify that it should mutate If nodes with assignments in their predicates
|
543
569
|
visitor.mutate("IfNode[predicate: Assign | OpAssign]") do |node|
|
@@ -567,20 +593,18 @@ SyntaxTree::Formatter.format(source, program.accept(visitor))
|
|
567
593
|
# => "if (a = 1)\nend\n"
|
568
594
|
```
|
569
595
|
|
570
|
-
###
|
596
|
+
### WithScope
|
571
597
|
|
572
|
-
The `
|
573
|
-
defined inside each environment. A `current_environment` accessor is made available to the request, allowing it to find
|
574
|
-
all usages and definitions of a local.
|
598
|
+
The `WithScope` module can be included in visitors to automatically keep track of local variables and arguments defined inside each scope. A `current_scope` accessor is made available to the request, allowing it to find all usages and definitions of a local.
|
575
599
|
|
576
600
|
```ruby
|
577
601
|
class MyVisitor < Visitor
|
578
|
-
|
602
|
+
prepend WithScope
|
579
603
|
|
580
604
|
def visit_ident(node)
|
581
605
|
# find_local will return a Local for any local variables or arguments
|
582
606
|
# present in the current environment or nil if the identifier is not a local
|
583
|
-
local =
|
607
|
+
local = current_scope.find_local(node)
|
584
608
|
|
585
609
|
puts local.type # the type of the local (:variable or :argument)
|
586
610
|
puts local.definitions # the array of locations where this local is defined
|
data/Rakefile
CHANGED
@@ -4,6 +4,8 @@ require "bundler/gem_tasks"
|
|
4
4
|
require "rake/testtask"
|
5
5
|
require "syntax_tree/rake_tasks"
|
6
6
|
|
7
|
+
Rake.add_rakelib "tasks"
|
8
|
+
|
7
9
|
Rake::TestTask.new(:test) do |t|
|
8
10
|
t.libs << "test"
|
9
11
|
t.libs << "lib"
|
@@ -14,7 +16,16 @@ task default: :test
|
|
14
16
|
|
15
17
|
configure = ->(task) do
|
16
18
|
task.source_files =
|
17
|
-
FileList[
|
19
|
+
FileList[
|
20
|
+
%w[
|
21
|
+
Gemfile
|
22
|
+
Rakefile
|
23
|
+
syntax_tree.gemspec
|
24
|
+
lib/**/*.rb
|
25
|
+
tasks/*.rake
|
26
|
+
test/*.rb
|
27
|
+
]
|
28
|
+
]
|
18
29
|
|
19
30
|
# Since Syntax Tree supports back to Ruby 2.7.0, we need to make sure that we
|
20
31
|
# format our code such that it's compatible with that version. This actually
|
@@ -26,10 +37,3 @@ end
|
|
26
37
|
|
27
38
|
SyntaxTree::Rake::CheckTask.new(&configure)
|
28
39
|
SyntaxTree::Rake::WriteTask.new(&configure)
|
29
|
-
|
30
|
-
desc "Run mspec tests using YARV emulation"
|
31
|
-
task :spec do
|
32
|
-
Dir["./spec/ruby/language/**/*_spec.rb"].each do |filepath|
|
33
|
-
sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}"
|
34
|
-
end
|
35
|
-
end
|
data/bin/console
CHANGED
data/bin/whitequark
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "parser/current"
|
6
|
+
|
7
|
+
$:.unshift(File.expand_path("../lib", __dir__))
|
8
|
+
require "syntax_tree"
|
9
|
+
|
10
|
+
# First, opt in to every AST feature.
|
11
|
+
Parser::Builders::Default.modernize
|
12
|
+
|
13
|
+
# Modify the source map == check so that it doesn't check against the node
|
14
|
+
# itself so we don't get into a recursive loop.
|
15
|
+
Parser::Source::Map.prepend(
|
16
|
+
Module.new {
|
17
|
+
def ==(other)
|
18
|
+
self.class == other.class &&
|
19
|
+
(instance_variables - %i[@node]).map do |ivar|
|
20
|
+
instance_variable_get(ivar) == other.instance_variable_get(ivar)
|
21
|
+
end.reduce(:&)
|
22
|
+
end
|
23
|
+
}
|
24
|
+
)
|
25
|
+
|
26
|
+
# Next, ensure that we're comparing the nodes and also comparing the source
|
27
|
+
# ranges so that we're getting all of the necessary information.
|
28
|
+
Parser::AST::Node.prepend(
|
29
|
+
Module.new {
|
30
|
+
def ==(other)
|
31
|
+
super && (location == other.location)
|
32
|
+
end
|
33
|
+
}
|
34
|
+
)
|
35
|
+
|
36
|
+
source = ARGF.read
|
37
|
+
|
38
|
+
parser = Parser::CurrentRuby.new
|
39
|
+
parser.diagnostics.all_errors_are_fatal = true
|
40
|
+
|
41
|
+
buffer = Parser::Source::Buffer.new("(string)", 1)
|
42
|
+
buffer.source = source.dup.force_encoding(parser.default_encoding)
|
43
|
+
|
44
|
+
stree = SyntaxTree::Translation.to_parser(SyntaxTree.parse(source), buffer)
|
45
|
+
ptree = parser.parse(buffer)
|
46
|
+
|
47
|
+
if stree == ptree
|
48
|
+
puts "Syntax trees are equivalent."
|
49
|
+
elsif stree.inspect == ptree.inspect
|
50
|
+
warn "Syntax tree locations are different."
|
51
|
+
|
52
|
+
queue = [[stree, ptree]]
|
53
|
+
while (left, right = queue.shift)
|
54
|
+
if left.location != right.location
|
55
|
+
warn "Different node:"
|
56
|
+
pp left
|
57
|
+
|
58
|
+
warn "Different location:"
|
59
|
+
|
60
|
+
warn "Syntax Tree:"
|
61
|
+
pp left.location
|
62
|
+
|
63
|
+
warn "whitequark/parser:"
|
64
|
+
pp right.location
|
65
|
+
|
66
|
+
exit
|
67
|
+
end
|
68
|
+
|
69
|
+
left.children.zip(right.children).each do |left_child, right_child|
|
70
|
+
queue << [left_child, right_child] if left_child.is_a?(Parser::AST::Node)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
else
|
74
|
+
warn "Syntax Tree:"
|
75
|
+
pp stree
|
76
|
+
|
77
|
+
warn "whitequark/parser:"
|
78
|
+
pp ptree
|
79
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# Changing structure
|
2
|
+
|
3
|
+
First and foremost, changing the structure of the tree in any way is a major breaking change. It forces the consumers to update their visitors, pattern matches, and method calls. It should not be taking lightly, and can only happen on a major version change. So keep that in mind.
|
4
|
+
|
5
|
+
That said, if you do want to change the structure of the tree, there are a few steps that you have to take. They are enumerated below.
|
6
|
+
|
7
|
+
1. Change the structure in the required node classes. This could mean adding/removing classes or adding/removing fields. Be sure to also update the `copy` and `===` methods to be sure that they are correct.
|
8
|
+
2. Update the parser to correctly create the new structure.
|
9
|
+
3. Update any visitor methods that are affected by the change. For example, if adding a new node make sure to create the new visit method alias in the `Visitor` class.
|
10
|
+
4. Update the `FieldVisitor` class to be sure that the various serializers, pretty printers, and matchers all get updated accordingly.
|
11
|
+
5. Update the `DSL` module to be sure that folks can correctly create nodes with the new structure.
|
12
|
+
6. Ensure the formatting of the code hasn't changed. This can mostly be done by running the tests, but if there's a corner case that we don't cover that is now exposed by your change be sure to add test cases.
|
13
|
+
7. Update the translation visitors to ensure we're still translating into other ASTs correctly.
|
14
|
+
8. Update the YARV compiler visitor to ensure we're still compiling correctly.
|
15
|
+
9. Make sure we aren't referencing the previous structure in any documentation or tests.
|
16
|
+
10. Be sure to update `CHANGELOG.md` with a description of the change that you made.
|
@@ -29,7 +29,7 @@ module SyntaxTree
|
|
29
29
|
def corrections
|
30
30
|
@corrections ||=
|
31
31
|
DidYouMean::SpellChecker.new(
|
32
|
-
dictionary:
|
32
|
+
dictionary: BasicVisitor.valid_visit_methods
|
33
33
|
).correct(visit_method)
|
34
34
|
end
|
35
35
|
|
@@ -40,7 +40,40 @@ module SyntaxTree
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
+
# This module is responsible for checking all of the methods defined within
|
44
|
+
# a given block to ensure that they are valid visit methods.
|
45
|
+
class VisitMethodsChecker < Module
|
46
|
+
Status = Struct.new(:checking)
|
47
|
+
|
48
|
+
# This is the status of the checker. It's used to determine whether or not
|
49
|
+
# we should be checking the methods that are defined. It is kept as an
|
50
|
+
# instance variable so that it can be disabled later.
|
51
|
+
attr_reader :status
|
52
|
+
|
53
|
+
def initialize
|
54
|
+
# We need the status to be an instance variable so that it can be
|
55
|
+
# accessed by the disable! method, but also a local variable so that it
|
56
|
+
# can be captured by the define_method block.
|
57
|
+
status = @status = Status.new(true)
|
58
|
+
|
59
|
+
define_method(:method_added) do |name|
|
60
|
+
BasicVisitor.visit_method(name) if status.checking
|
61
|
+
super(name)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def disable!
|
66
|
+
status.checking = false
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
43
70
|
class << self
|
71
|
+
# This is the list of all of the valid visit methods.
|
72
|
+
def valid_visit_methods
|
73
|
+
@valid_visit_methods ||=
|
74
|
+
Visitor.instance_methods.grep(/^visit_(?!child_nodes)/)
|
75
|
+
end
|
76
|
+
|
44
77
|
# This method is here to help folks write visitors.
|
45
78
|
#
|
46
79
|
# It's not always easy to ensure you're writing the correct method name in
|
@@ -51,15 +84,21 @@ module SyntaxTree
|
|
51
84
|
# name. It will raise an error if the visit method you're defining isn't
|
52
85
|
# actually a method on the parent visitor.
|
53
86
|
def visit_method(method_name)
|
54
|
-
return if
|
87
|
+
return if valid_visit_methods.include?(method_name)
|
55
88
|
|
56
89
|
raise VisitMethodError, method_name
|
57
90
|
end
|
58
91
|
|
59
|
-
# This is
|
92
|
+
# This method is here to help folks write visitors.
|
93
|
+
#
|
94
|
+
# Within the given block, every method that is defined will be checked to
|
95
|
+
# ensure it's a valid visit method using the BasicVisitor::visit_method
|
96
|
+
# method defined above.
|
60
97
|
def visit_methods
|
61
|
-
|
62
|
-
|
98
|
+
checker = VisitMethodsChecker.new
|
99
|
+
extend(checker)
|
100
|
+
yield
|
101
|
+
checker.disable!
|
63
102
|
end
|
64
103
|
end
|
65
104
|
|
data/lib/syntax_tree/cli.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "etc"
|
3
4
|
require "optparse"
|
4
5
|
|
5
6
|
module SyntaxTree
|
@@ -238,7 +239,7 @@ module SyntaxTree
|
|
238
239
|
# representation.
|
239
240
|
class Json < Action
|
240
241
|
def run(item)
|
241
|
-
object =
|
242
|
+
object = item.handler.parse(item.source).accept(JSONVisitor.new)
|
242
243
|
puts JSON.pretty_generate(object)
|
243
244
|
end
|
244
245
|
end
|
@@ -501,7 +502,6 @@ module SyntaxTree
|
|
501
502
|
when "j", "json"
|
502
503
|
Json.new(options)
|
503
504
|
when "lsp"
|
504
|
-
require "syntax_tree/language_server"
|
505
505
|
LanguageServer.new(print_width: options.print_width).run
|
506
506
|
return 0
|
507
507
|
when "m", "match"
|
data/lib/syntax_tree/dsl.rb
CHANGED
@@ -210,12 +210,17 @@ module SyntaxTree
|
|
210
210
|
end
|
211
211
|
|
212
212
|
# Create a new ClassDeclaration node.
|
213
|
-
def ClassDeclaration(
|
213
|
+
def ClassDeclaration(
|
214
|
+
constant,
|
215
|
+
superclass,
|
216
|
+
bodystmt,
|
217
|
+
location = Location.default
|
218
|
+
)
|
214
219
|
ClassDeclaration.new(
|
215
220
|
constant: constant,
|
216
221
|
superclass: superclass,
|
217
222
|
bodystmt: bodystmt,
|
218
|
-
location:
|
223
|
+
location: location
|
219
224
|
)
|
220
225
|
end
|
221
226
|
|
@@ -225,12 +230,12 @@ module SyntaxTree
|
|
225
230
|
end
|
226
231
|
|
227
232
|
# Create a new Command node.
|
228
|
-
def Command(message, arguments, block)
|
233
|
+
def Command(message, arguments, block, location = Location.default)
|
229
234
|
Command.new(
|
230
235
|
message: message,
|
231
236
|
arguments: arguments,
|
232
237
|
block: block,
|
233
|
-
location:
|
238
|
+
location: location
|
234
239
|
)
|
235
240
|
end
|
236
241
|
|
@@ -247,8 +252,8 @@ module SyntaxTree
|
|
247
252
|
end
|
248
253
|
|
249
254
|
# Create a new Comment node.
|
250
|
-
def Comment(value, inline)
|
251
|
-
Comment.new(value: value, inline: inline, location:
|
255
|
+
def Comment(value, inline, location = Location.default)
|
256
|
+
Comment.new(value: value, inline: inline, location: location)
|
252
257
|
end
|
253
258
|
|
254
259
|
# Create a new Const node.
|
@@ -285,14 +290,21 @@ module SyntaxTree
|
|
285
290
|
end
|
286
291
|
|
287
292
|
# Create a new DefNode node.
|
288
|
-
def DefNode(
|
293
|
+
def DefNode(
|
294
|
+
target,
|
295
|
+
operator,
|
296
|
+
name,
|
297
|
+
params,
|
298
|
+
bodystmt,
|
299
|
+
location = Location.default
|
300
|
+
)
|
289
301
|
DefNode.new(
|
290
302
|
target: target,
|
291
303
|
operator: operator,
|
292
304
|
name: name,
|
293
305
|
params: params,
|
294
306
|
bodystmt: bodystmt,
|
295
|
-
location:
|
307
|
+
location: location
|
296
308
|
)
|
297
309
|
end
|
298
310
|
|
@@ -565,8 +577,8 @@ module SyntaxTree
|
|
565
577
|
end
|
566
578
|
|
567
579
|
# Create a new MethodAddBlock node.
|
568
|
-
def MethodAddBlock(call, block)
|
569
|
-
MethodAddBlock.new(call: call, block: block, location:
|
580
|
+
def MethodAddBlock(call, block, location = Location.default)
|
581
|
+
MethodAddBlock.new(call: call, block: block, location: location)
|
570
582
|
end
|
571
583
|
|
572
584
|
# Create a new MLHS node.
|
@@ -779,7 +791,7 @@ module SyntaxTree
|
|
779
791
|
|
780
792
|
# Create a new Statements node.
|
781
793
|
def Statements(body)
|
782
|
-
Statements.new(
|
794
|
+
Statements.new(body: body, location: Location.default)
|
783
795
|
end
|
784
796
|
|
785
797
|
# Create a new StringContent node.
|