syntax_tree 5.3.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +12 -1
  3. data/CHANGELOG.md +64 -1
  4. data/Gemfile.lock +2 -2
  5. data/README.md +28 -9
  6. data/Rakefile +12 -8
  7. data/bin/console +1 -0
  8. data/bin/whitequark +79 -0
  9. data/doc/changing_structure.md +16 -0
  10. data/lib/syntax_tree/basic_visitor.rb +44 -5
  11. data/lib/syntax_tree/cli.rb +2 -2
  12. data/lib/syntax_tree/dsl.rb +23 -11
  13. data/lib/syntax_tree/{visitor/field_visitor.rb → field_visitor.rb} +54 -55
  14. data/lib/syntax_tree/formatter.rb +1 -1
  15. data/lib/syntax_tree/index.rb +56 -54
  16. data/lib/syntax_tree/json_visitor.rb +55 -0
  17. data/lib/syntax_tree/language_server.rb +157 -2
  18. data/lib/syntax_tree/match_visitor.rb +120 -0
  19. data/lib/syntax_tree/mermaid.rb +177 -0
  20. data/lib/syntax_tree/mermaid_visitor.rb +69 -0
  21. data/lib/syntax_tree/{visitor/mutation_visitor.rb → mutation_visitor.rb} +27 -27
  22. data/lib/syntax_tree/node.rb +198 -107
  23. data/lib/syntax_tree/parser.rb +322 -118
  24. data/lib/syntax_tree/pretty_print_visitor.rb +83 -0
  25. data/lib/syntax_tree/reflection.rb +241 -0
  26. data/lib/syntax_tree/translation/parser.rb +3019 -0
  27. data/lib/syntax_tree/translation/rubocop_ast.rb +21 -0
  28. data/lib/syntax_tree/translation.rb +28 -0
  29. data/lib/syntax_tree/version.rb +1 -1
  30. data/lib/syntax_tree/with_scope.rb +244 -0
  31. data/lib/syntax_tree/yarv/basic_block.rb +53 -0
  32. data/lib/syntax_tree/yarv/calldata.rb +91 -0
  33. data/lib/syntax_tree/yarv/compiler.rb +110 -100
  34. data/lib/syntax_tree/yarv/control_flow_graph.rb +257 -0
  35. data/lib/syntax_tree/yarv/data_flow_graph.rb +338 -0
  36. data/lib/syntax_tree/yarv/decompiler.rb +1 -1
  37. data/lib/syntax_tree/yarv/disassembler.rb +104 -80
  38. data/lib/syntax_tree/yarv/instruction_sequence.rb +43 -18
  39. data/lib/syntax_tree/yarv/instructions.rb +203 -649
  40. data/lib/syntax_tree/yarv/legacy.rb +12 -24
  41. data/lib/syntax_tree/yarv/sea_of_nodes.rb +534 -0
  42. data/lib/syntax_tree/yarv.rb +18 -0
  43. data/lib/syntax_tree.rb +88 -56
  44. data/tasks/sorbet.rake +277 -0
  45. data/tasks/whitequark.rake +87 -0
  46. metadata +23 -11
  47. data/.gitmodules +0 -9
  48. data/lib/syntax_tree/language_server/inlay_hints.rb +0 -159
  49. data/lib/syntax_tree/visitor/environment.rb +0 -84
  50. data/lib/syntax_tree/visitor/json_visitor.rb +0 -55
  51. data/lib/syntax_tree/visitor/match_visitor.rb +0 -122
  52. data/lib/syntax_tree/visitor/pretty_print_visitor.rb +0 -85
  53. data/lib/syntax_tree/visitor/with_environment.rb +0 -140
@@ -1,159 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class LanguageServer
5
- # This class provides inlay hints for the language server. For more
6
- # information, see the spec here:
7
- # https://github.com/microsoft/language-server-protocol/issues/956.
8
- class InlayHints < Visitor
9
- # This represents a hint that is going to be displayed in the editor.
10
- class Hint
11
- attr_reader :line, :character, :label
12
-
13
- def initialize(line:, character:, label:)
14
- @line = line
15
- @character = character
16
- @label = label
17
- end
18
-
19
- # This is the shape that the LSP expects.
20
- def to_json(*opts)
21
- {
22
- position: {
23
- line: line,
24
- character: character
25
- },
26
- label: label
27
- }.to_json(*opts)
28
- end
29
- end
30
-
31
- attr_reader :stack, :hints
32
-
33
- def initialize
34
- @stack = []
35
- @hints = []
36
- end
37
-
38
- def visit(node)
39
- stack << node
40
- result = super
41
- stack.pop
42
- result
43
- end
44
-
45
- # Adds parentheses around assignments contained within the default values
46
- # of parameters. For example,
47
- #
48
- # def foo(a = b = c)
49
- # end
50
- #
51
- # becomes
52
- #
53
- # def foo(a = ₍b = c₎)
54
- # end
55
- #
56
- def visit_assign(node)
57
- parentheses(node.location) if stack[-2].is_a?(Params)
58
- super
59
- end
60
-
61
- # Adds parentheses around binary expressions to make it clear which
62
- # subexpression will be evaluated first. For example,
63
- #
64
- # a + b * c
65
- #
66
- # becomes
67
- #
68
- # a + ₍b * c₎
69
- #
70
- def visit_binary(node)
71
- case stack[-2]
72
- when Assign, OpAssign
73
- parentheses(node.location)
74
- when Binary
75
- parentheses(node.location) if stack[-2].operator != node.operator
76
- end
77
-
78
- super
79
- end
80
-
81
- # Adds parentheses around ternary operators contained within certain
82
- # expressions where it could be confusing which subexpression will get
83
- # evaluated first. For example,
84
- #
85
- # a ? b : c ? d : e
86
- #
87
- # becomes
88
- #
89
- # a ? b : ₍c ? d : e₎
90
- #
91
- def visit_if_op(node)
92
- case stack[-2]
93
- when Assign, Binary, IfOp, OpAssign
94
- parentheses(node.location)
95
- end
96
-
97
- super
98
- end
99
-
100
- # Adds the implicitly rescued StandardError into a bare rescue clause. For
101
- # example,
102
- #
103
- # begin
104
- # rescue
105
- # end
106
- #
107
- # becomes
108
- #
109
- # begin
110
- # rescue StandardError
111
- # end
112
- #
113
- def visit_rescue(node)
114
- if node.exception.nil?
115
- hints << Hint.new(
116
- line: node.location.start_line - 1,
117
- character: node.location.start_column + "rescue".length,
118
- label: " StandardError"
119
- )
120
- end
121
-
122
- super
123
- end
124
-
125
- # Adds parentheses around unary statements using the - operator that are
126
- # contained within Binary nodes. For example,
127
- #
128
- # -a + b
129
- #
130
- # becomes
131
- #
132
- # ₍-a₎ + b
133
- #
134
- def visit_unary(node)
135
- if stack[-2].is_a?(Binary) && (node.operator == "-")
136
- parentheses(node.location)
137
- end
138
-
139
- super
140
- end
141
-
142
- private
143
-
144
- def parentheses(location)
145
- hints << Hint.new(
146
- line: location.start_line - 1,
147
- character: location.start_column,
148
- label: "₍"
149
- )
150
-
151
- hints << Hint.new(
152
- line: location.end_line - 1,
153
- character: location.end_column,
154
- label: "₎"
155
- )
156
- end
157
- end
158
- end
159
- end
@@ -1,84 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- # The environment class is used to keep track of local variables and arguments
5
- # inside a particular scope
6
- class Environment
7
- # This class tracks the occurrences of a local variable or argument
8
- class Local
9
- # [Symbol] The type of the local (e.g. :argument, :variable)
10
- attr_reader :type
11
-
12
- # [Array[Location]] The locations of all definitions and assignments of
13
- # this local
14
- attr_reader :definitions
15
-
16
- # [Array[Location]] The locations of all usages of this local
17
- attr_reader :usages
18
-
19
- # initialize: (Symbol type) -> void
20
- def initialize(type)
21
- @type = type
22
- @definitions = []
23
- @usages = []
24
- end
25
-
26
- # add_definition: (Location location) -> void
27
- def add_definition(location)
28
- @definitions << location
29
- end
30
-
31
- # add_usage: (Location location) -> void
32
- def add_usage(location)
33
- @usages << location
34
- end
35
- end
36
-
37
- # [Array[Local]] The local variables and arguments defined in this
38
- # environment
39
- attr_reader :locals
40
-
41
- # [Environment | nil] The parent environment
42
- attr_reader :parent
43
-
44
- # initialize: (Environment | nil parent) -> void
45
- def initialize(parent = nil)
46
- @locals = {}
47
- @parent = parent
48
- end
49
-
50
- # Adding a local definition will either insert a new entry in the locals
51
- # hash or append a new definition location to an existing local. Notice that
52
- # it's not possible to change the type of a local after it has been
53
- # registered
54
- # add_local_definition: (Ident | Label identifier, Symbol type) -> void
55
- def add_local_definition(identifier, type)
56
- name = identifier.value.delete_suffix(":")
57
-
58
- @locals[name] ||= Local.new(type)
59
- @locals[name].add_definition(identifier.location)
60
- end
61
-
62
- # Adding a local usage will either insert a new entry in the locals
63
- # hash or append a new usage location to an existing local. Notice that
64
- # it's not possible to change the type of a local after it has been
65
- # registered
66
- # add_local_usage: (Ident | Label identifier, Symbol type) -> void
67
- def add_local_usage(identifier, type)
68
- name = identifier.value.delete_suffix(":")
69
-
70
- @locals[name] ||= Local.new(type)
71
- @locals[name].add_usage(identifier.location)
72
- end
73
-
74
- # Try to find the local given its name in this environment or any of its
75
- # parents
76
- # find_local: (String name) -> Local | nil
77
- def find_local(name)
78
- local = @locals[name]
79
- return local unless local.nil?
80
-
81
- @parent&.find_local(name)
82
- end
83
- end
84
- end
@@ -1,55 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor transforms the AST into a hash that contains only primitives
6
- # that can be easily serialized into JSON.
7
- class JSONVisitor < FieldVisitor
8
- attr_reader :target
9
-
10
- def initialize
11
- @target = nil
12
- end
13
-
14
- private
15
-
16
- def comments(node)
17
- target[:comments] = visit_all(node.comments)
18
- end
19
-
20
- def field(name, value)
21
- target[name] = value.is_a?(Node) ? visit(value) : value
22
- end
23
-
24
- def list(name, values)
25
- target[name] = visit_all(values)
26
- end
27
-
28
- def node(node, type)
29
- previous = @target
30
- @target = { type: type, location: visit_location(node.location) }
31
- yield
32
- @target
33
- ensure
34
- @target = previous
35
- end
36
-
37
- def pairs(name, values)
38
- target[name] = values.map { |(key, value)| [visit(key), visit(value)] }
39
- end
40
-
41
- def text(name, value)
42
- target[name] = value
43
- end
44
-
45
- def visit_location(location)
46
- [
47
- location.start_line,
48
- location.start_char,
49
- location.end_line,
50
- location.end_char
51
- ]
52
- end
53
- end
54
- end
55
- end
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor transforms the AST into a Ruby pattern matching expression
6
- # that would match correctly against the AST.
7
- class MatchVisitor < FieldVisitor
8
- attr_reader :q
9
-
10
- def initialize(q)
11
- @q = q
12
- end
13
-
14
- def visit(node)
15
- case node
16
- when Node
17
- super
18
- when String
19
- # pp will split up a string on newlines and concat them together using
20
- # a "+" operator. This breaks the pattern matching expression. So
21
- # instead we're going to check here for strings and manually put the
22
- # entire value into the output buffer.
23
- q.text(node.inspect)
24
- else
25
- node.pretty_print(q)
26
- end
27
- end
28
-
29
- private
30
-
31
- def comments(node)
32
- return if node.comments.empty?
33
-
34
- q.nest(0) do
35
- q.text("comments: [")
36
- q.indent do
37
- q.breakable("")
38
- q.seplist(node.comments) { |comment| visit(comment) }
39
- end
40
- q.breakable("")
41
- q.text("]")
42
- end
43
- end
44
-
45
- def field(name, value)
46
- q.nest(0) do
47
- q.text(name)
48
- q.text(": ")
49
- visit(value)
50
- end
51
- end
52
-
53
- def list(name, values)
54
- q.group do
55
- q.text(name)
56
- q.text(": [")
57
- q.indent do
58
- q.breakable("")
59
- q.seplist(values) { |value| visit(value) }
60
- end
61
- q.breakable("")
62
- q.text("]")
63
- end
64
- end
65
-
66
- def node(node, _type)
67
- items = []
68
- q.with_target(items) { yield }
69
-
70
- if items.empty?
71
- q.text(node.class.name)
72
- return
73
- end
74
-
75
- q.group do
76
- q.text(node.class.name)
77
- q.text("[")
78
- q.indent do
79
- q.breakable("")
80
- q.seplist(items) { |item| q.target << item }
81
- end
82
- q.breakable("")
83
- q.text("]")
84
- end
85
- end
86
-
87
- def pairs(name, values)
88
- q.group do
89
- q.text(name)
90
- q.text(": [")
91
- q.indent do
92
- q.breakable("")
93
- q.seplist(values) do |(key, value)|
94
- q.group do
95
- q.text("[")
96
- q.indent do
97
- q.breakable("")
98
- visit(key)
99
- q.text(",")
100
- q.breakable
101
- visit(value || nil)
102
- end
103
- q.breakable("")
104
- q.text("]")
105
- end
106
- end
107
- end
108
- q.breakable("")
109
- q.text("]")
110
- end
111
- end
112
-
113
- def text(name, value)
114
- q.nest(0) do
115
- q.text(name)
116
- q.text(": ")
117
- value.pretty_print(q)
118
- end
119
- end
120
- end
121
- end
122
- end
@@ -1,85 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor pretty-prints the AST into an equivalent s-expression.
6
- class PrettyPrintVisitor < FieldVisitor
7
- attr_reader :q
8
-
9
- def initialize(q)
10
- @q = q
11
- end
12
-
13
- # This is here because we need to make sure the operator is cast to a
14
- # string before we print it out.
15
- def visit_binary(node)
16
- node(node, "binary") do
17
- field("left", node.left)
18
- text("operator", node.operator.to_s)
19
- field("right", node.right)
20
- comments(node)
21
- end
22
- end
23
-
24
- # This is here to make it a little nicer to look at labels since they
25
- # typically have their : at the end of the value.
26
- def visit_label(node)
27
- node(node, "label") do
28
- q.breakable
29
- q.text(":")
30
- q.text(node.value[0...-1])
31
- comments(node)
32
- end
33
- end
34
-
35
- private
36
-
37
- def comments(node)
38
- return if node.comments.empty?
39
-
40
- q.breakable
41
- q.group(2, "(", ")") do
42
- q.seplist(node.comments) { |comment| q.pp(comment) }
43
- end
44
- end
45
-
46
- def field(_name, value)
47
- q.breakable
48
- q.pp(value)
49
- end
50
-
51
- def list(_name, values)
52
- q.breakable
53
- q.group(2, "(", ")") { q.seplist(values) { |value| q.pp(value) } }
54
- end
55
-
56
- def node(_node, type)
57
- q.group(2, "(", ")") do
58
- q.text(type)
59
- yield
60
- end
61
- end
62
-
63
- def pairs(_name, values)
64
- q.group(2, "(", ")") do
65
- q.seplist(values) do |(key, value)|
66
- q.pp(key)
67
-
68
- if value
69
- q.text("=")
70
- q.group(2) do
71
- q.breakable("")
72
- q.pp(value)
73
- end
74
- end
75
- end
76
- end
77
- end
78
-
79
- def text(_name, value)
80
- q.breakable
81
- q.text(value)
82
- end
83
- end
84
- end
85
- end
@@ -1,140 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- # WithEnvironment is a module intended to be included in classes inheriting
5
- # from Visitor. The module overrides a few visit methods to automatically keep
6
- # track of local variables and arguments defined in the current environment.
7
- # Example usage:
8
- # class MyVisitor < Visitor
9
- # include WithEnvironment
10
- #
11
- # def visit_ident(node)
12
- # # Check if we're visiting an identifier for an argument, a local
13
- # variable or something else
14
- # local = current_environment.find_local(node)
15
- #
16
- # if local.type == :argument
17
- # # handle identifiers for arguments
18
- # elsif local.type == :variable
19
- # # handle identifiers for variables
20
- # else
21
- # # handle other identifiers, such as method names
22
- # end
23
- # end
24
- module WithEnvironment
25
- def current_environment
26
- @current_environment ||= Environment.new
27
- end
28
-
29
- def with_new_environment
30
- previous_environment = @current_environment
31
- @current_environment = Environment.new(previous_environment)
32
- yield
33
- ensure
34
- @current_environment = previous_environment
35
- end
36
-
37
- # Visits for nodes that create new environments, such as classes, modules
38
- # and method definitions
39
- def visit_class(node)
40
- with_new_environment { super }
41
- end
42
-
43
- def visit_module(node)
44
- with_new_environment { super }
45
- end
46
-
47
- # When we find a method invocation with a block, only the code that happens
48
- # inside of the block needs a fresh environment. The method invocation
49
- # itself happens in the same environment
50
- def visit_method_add_block(node)
51
- visit(node.call)
52
- with_new_environment { visit(node.block) }
53
- end
54
-
55
- def visit_def(node)
56
- with_new_environment { super }
57
- end
58
-
59
- # Visit for keeping track of local arguments, such as method and block
60
- # arguments
61
- def visit_params(node)
62
- add_argument_definitions(node.requireds)
63
-
64
- node.posts.each do |param|
65
- current_environment.add_local_definition(param, :argument)
66
- end
67
-
68
- node.keywords.each do |param|
69
- current_environment.add_local_definition(param.first, :argument)
70
- end
71
-
72
- node.optionals.each do |param|
73
- current_environment.add_local_definition(param.first, :argument)
74
- end
75
-
76
- super
77
- end
78
-
79
- def visit_rest_param(node)
80
- name = node.name
81
- current_environment.add_local_definition(name, :argument) if name
82
-
83
- super
84
- end
85
-
86
- def visit_kwrest_param(node)
87
- name = node.name
88
- current_environment.add_local_definition(name, :argument) if name
89
-
90
- super
91
- end
92
-
93
- def visit_blockarg(node)
94
- name = node.name
95
- current_environment.add_local_definition(name, :argument) if name
96
-
97
- super
98
- end
99
-
100
- # Visit for keeping track of local variable definitions
101
- def visit_var_field(node)
102
- value = node.value
103
-
104
- if value.is_a?(SyntaxTree::Ident)
105
- current_environment.add_local_definition(value, :variable)
106
- end
107
-
108
- super
109
- end
110
-
111
- alias visit_pinned_var_ref visit_var_field
112
-
113
- # Visits for keeping track of variable and argument usages
114
- def visit_var_ref(node)
115
- value = node.value
116
-
117
- if value.is_a?(SyntaxTree::Ident)
118
- definition = current_environment.find_local(value.value)
119
-
120
- if definition
121
- current_environment.add_local_usage(value, definition.type)
122
- end
123
- end
124
-
125
- super
126
- end
127
-
128
- private
129
-
130
- def add_argument_definitions(list)
131
- list.each do |param|
132
- if param.is_a?(SyntaxTree::MLHSParen)
133
- add_argument_definitions(param.contents.parts)
134
- else
135
- current_environment.add_local_definition(param, :argument)
136
- end
137
- end
138
- end
139
- end
140
- end