syntax_tree 5.3.0 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +12 -1
  3. data/CHANGELOG.md +64 -1
  4. data/Gemfile.lock +2 -2
  5. data/README.md +28 -9
  6. data/Rakefile +12 -8
  7. data/bin/console +1 -0
  8. data/bin/whitequark +79 -0
  9. data/doc/changing_structure.md +16 -0
  10. data/lib/syntax_tree/basic_visitor.rb +44 -5
  11. data/lib/syntax_tree/cli.rb +2 -2
  12. data/lib/syntax_tree/dsl.rb +23 -11
  13. data/lib/syntax_tree/{visitor/field_visitor.rb → field_visitor.rb} +54 -55
  14. data/lib/syntax_tree/formatter.rb +1 -1
  15. data/lib/syntax_tree/index.rb +56 -54
  16. data/lib/syntax_tree/json_visitor.rb +55 -0
  17. data/lib/syntax_tree/language_server.rb +157 -2
  18. data/lib/syntax_tree/match_visitor.rb +120 -0
  19. data/lib/syntax_tree/mermaid.rb +177 -0
  20. data/lib/syntax_tree/mermaid_visitor.rb +69 -0
  21. data/lib/syntax_tree/{visitor/mutation_visitor.rb → mutation_visitor.rb} +27 -27
  22. data/lib/syntax_tree/node.rb +198 -107
  23. data/lib/syntax_tree/parser.rb +322 -118
  24. data/lib/syntax_tree/pretty_print_visitor.rb +83 -0
  25. data/lib/syntax_tree/reflection.rb +241 -0
  26. data/lib/syntax_tree/translation/parser.rb +3019 -0
  27. data/lib/syntax_tree/translation/rubocop_ast.rb +21 -0
  28. data/lib/syntax_tree/translation.rb +28 -0
  29. data/lib/syntax_tree/version.rb +1 -1
  30. data/lib/syntax_tree/with_scope.rb +244 -0
  31. data/lib/syntax_tree/yarv/basic_block.rb +53 -0
  32. data/lib/syntax_tree/yarv/calldata.rb +91 -0
  33. data/lib/syntax_tree/yarv/compiler.rb +110 -100
  34. data/lib/syntax_tree/yarv/control_flow_graph.rb +257 -0
  35. data/lib/syntax_tree/yarv/data_flow_graph.rb +338 -0
  36. data/lib/syntax_tree/yarv/decompiler.rb +1 -1
  37. data/lib/syntax_tree/yarv/disassembler.rb +104 -80
  38. data/lib/syntax_tree/yarv/instruction_sequence.rb +43 -18
  39. data/lib/syntax_tree/yarv/instructions.rb +203 -649
  40. data/lib/syntax_tree/yarv/legacy.rb +12 -24
  41. data/lib/syntax_tree/yarv/sea_of_nodes.rb +534 -0
  42. data/lib/syntax_tree/yarv.rb +18 -0
  43. data/lib/syntax_tree.rb +88 -56
  44. data/tasks/sorbet.rake +277 -0
  45. data/tasks/whitequark.rake +87 -0
  46. metadata +23 -11
  47. data/.gitmodules +0 -9
  48. data/lib/syntax_tree/language_server/inlay_hints.rb +0 -159
  49. data/lib/syntax_tree/visitor/environment.rb +0 -84
  50. data/lib/syntax_tree/visitor/json_visitor.rb +0 -55
  51. data/lib/syntax_tree/visitor/match_visitor.rb +0 -122
  52. data/lib/syntax_tree/visitor/pretty_print_visitor.rb +0 -85
  53. data/lib/syntax_tree/visitor/with_environment.rb +0 -140
@@ -1,159 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class LanguageServer
5
- # This class provides inlay hints for the language server. For more
6
- # information, see the spec here:
7
- # https://github.com/microsoft/language-server-protocol/issues/956.
8
- class InlayHints < Visitor
9
- # This represents a hint that is going to be displayed in the editor.
10
- class Hint
11
- attr_reader :line, :character, :label
12
-
13
- def initialize(line:, character:, label:)
14
- @line = line
15
- @character = character
16
- @label = label
17
- end
18
-
19
- # This is the shape that the LSP expects.
20
- def to_json(*opts)
21
- {
22
- position: {
23
- line: line,
24
- character: character
25
- },
26
- label: label
27
- }.to_json(*opts)
28
- end
29
- end
30
-
31
- attr_reader :stack, :hints
32
-
33
- def initialize
34
- @stack = []
35
- @hints = []
36
- end
37
-
38
- def visit(node)
39
- stack << node
40
- result = super
41
- stack.pop
42
- result
43
- end
44
-
45
- # Adds parentheses around assignments contained within the default values
46
- # of parameters. For example,
47
- #
48
- # def foo(a = b = c)
49
- # end
50
- #
51
- # becomes
52
- #
53
- # def foo(a = ₍b = c₎)
54
- # end
55
- #
56
- def visit_assign(node)
57
- parentheses(node.location) if stack[-2].is_a?(Params)
58
- super
59
- end
60
-
61
- # Adds parentheses around binary expressions to make it clear which
62
- # subexpression will be evaluated first. For example,
63
- #
64
- # a + b * c
65
- #
66
- # becomes
67
- #
68
- # a + ₍b * c₎
69
- #
70
- def visit_binary(node)
71
- case stack[-2]
72
- when Assign, OpAssign
73
- parentheses(node.location)
74
- when Binary
75
- parentheses(node.location) if stack[-2].operator != node.operator
76
- end
77
-
78
- super
79
- end
80
-
81
- # Adds parentheses around ternary operators contained within certain
82
- # expressions where it could be confusing which subexpression will get
83
- # evaluated first. For example,
84
- #
85
- # a ? b : c ? d : e
86
- #
87
- # becomes
88
- #
89
- # a ? b : ₍c ? d : e₎
90
- #
91
- def visit_if_op(node)
92
- case stack[-2]
93
- when Assign, Binary, IfOp, OpAssign
94
- parentheses(node.location)
95
- end
96
-
97
- super
98
- end
99
-
100
- # Adds the implicitly rescued StandardError into a bare rescue clause. For
101
- # example,
102
- #
103
- # begin
104
- # rescue
105
- # end
106
- #
107
- # becomes
108
- #
109
- # begin
110
- # rescue StandardError
111
- # end
112
- #
113
- def visit_rescue(node)
114
- if node.exception.nil?
115
- hints << Hint.new(
116
- line: node.location.start_line - 1,
117
- character: node.location.start_column + "rescue".length,
118
- label: " StandardError"
119
- )
120
- end
121
-
122
- super
123
- end
124
-
125
- # Adds parentheses around unary statements using the - operator that are
126
- # contained within Binary nodes. For example,
127
- #
128
- # -a + b
129
- #
130
- # becomes
131
- #
132
- # ₍-a₎ + b
133
- #
134
- def visit_unary(node)
135
- if stack[-2].is_a?(Binary) && (node.operator == "-")
136
- parentheses(node.location)
137
- end
138
-
139
- super
140
- end
141
-
142
- private
143
-
144
- def parentheses(location)
145
- hints << Hint.new(
146
- line: location.start_line - 1,
147
- character: location.start_column,
148
- label: "₍"
149
- )
150
-
151
- hints << Hint.new(
152
- line: location.end_line - 1,
153
- character: location.end_column,
154
- label: "₎"
155
- )
156
- end
157
- end
158
- end
159
- end
@@ -1,84 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- # The environment class is used to keep track of local variables and arguments
5
- # inside a particular scope
6
- class Environment
7
- # This class tracks the occurrences of a local variable or argument
8
- class Local
9
- # [Symbol] The type of the local (e.g. :argument, :variable)
10
- attr_reader :type
11
-
12
- # [Array[Location]] The locations of all definitions and assignments of
13
- # this local
14
- attr_reader :definitions
15
-
16
- # [Array[Location]] The locations of all usages of this local
17
- attr_reader :usages
18
-
19
- # initialize: (Symbol type) -> void
20
- def initialize(type)
21
- @type = type
22
- @definitions = []
23
- @usages = []
24
- end
25
-
26
- # add_definition: (Location location) -> void
27
- def add_definition(location)
28
- @definitions << location
29
- end
30
-
31
- # add_usage: (Location location) -> void
32
- def add_usage(location)
33
- @usages << location
34
- end
35
- end
36
-
37
- # [Array[Local]] The local variables and arguments defined in this
38
- # environment
39
- attr_reader :locals
40
-
41
- # [Environment | nil] The parent environment
42
- attr_reader :parent
43
-
44
- # initialize: (Environment | nil parent) -> void
45
- def initialize(parent = nil)
46
- @locals = {}
47
- @parent = parent
48
- end
49
-
50
- # Adding a local definition will either insert a new entry in the locals
51
- # hash or append a new definition location to an existing local. Notice that
52
- # it's not possible to change the type of a local after it has been
53
- # registered
54
- # add_local_definition: (Ident | Label identifier, Symbol type) -> void
55
- def add_local_definition(identifier, type)
56
- name = identifier.value.delete_suffix(":")
57
-
58
- @locals[name] ||= Local.new(type)
59
- @locals[name].add_definition(identifier.location)
60
- end
61
-
62
- # Adding a local usage will either insert a new entry in the locals
63
- # hash or append a new usage location to an existing local. Notice that
64
- # it's not possible to change the type of a local after it has been
65
- # registered
66
- # add_local_usage: (Ident | Label identifier, Symbol type) -> void
67
- def add_local_usage(identifier, type)
68
- name = identifier.value.delete_suffix(":")
69
-
70
- @locals[name] ||= Local.new(type)
71
- @locals[name].add_usage(identifier.location)
72
- end
73
-
74
- # Try to find the local given its name in this environment or any of its
75
- # parents
76
- # find_local: (String name) -> Local | nil
77
- def find_local(name)
78
- local = @locals[name]
79
- return local unless local.nil?
80
-
81
- @parent&.find_local(name)
82
- end
83
- end
84
- end
@@ -1,55 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor transforms the AST into a hash that contains only primitives
6
- # that can be easily serialized into JSON.
7
- class JSONVisitor < FieldVisitor
8
- attr_reader :target
9
-
10
- def initialize
11
- @target = nil
12
- end
13
-
14
- private
15
-
16
- def comments(node)
17
- target[:comments] = visit_all(node.comments)
18
- end
19
-
20
- def field(name, value)
21
- target[name] = value.is_a?(Node) ? visit(value) : value
22
- end
23
-
24
- def list(name, values)
25
- target[name] = visit_all(values)
26
- end
27
-
28
- def node(node, type)
29
- previous = @target
30
- @target = { type: type, location: visit_location(node.location) }
31
- yield
32
- @target
33
- ensure
34
- @target = previous
35
- end
36
-
37
- def pairs(name, values)
38
- target[name] = values.map { |(key, value)| [visit(key), visit(value)] }
39
- end
40
-
41
- def text(name, value)
42
- target[name] = value
43
- end
44
-
45
- def visit_location(location)
46
- [
47
- location.start_line,
48
- location.start_char,
49
- location.end_line,
50
- location.end_char
51
- ]
52
- end
53
- end
54
- end
55
- end
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor transforms the AST into a Ruby pattern matching expression
6
- # that would match correctly against the AST.
7
- class MatchVisitor < FieldVisitor
8
- attr_reader :q
9
-
10
- def initialize(q)
11
- @q = q
12
- end
13
-
14
- def visit(node)
15
- case node
16
- when Node
17
- super
18
- when String
19
- # pp will split up a string on newlines and concat them together using
20
- # a "+" operator. This breaks the pattern matching expression. So
21
- # instead we're going to check here for strings and manually put the
22
- # entire value into the output buffer.
23
- q.text(node.inspect)
24
- else
25
- node.pretty_print(q)
26
- end
27
- end
28
-
29
- private
30
-
31
- def comments(node)
32
- return if node.comments.empty?
33
-
34
- q.nest(0) do
35
- q.text("comments: [")
36
- q.indent do
37
- q.breakable("")
38
- q.seplist(node.comments) { |comment| visit(comment) }
39
- end
40
- q.breakable("")
41
- q.text("]")
42
- end
43
- end
44
-
45
- def field(name, value)
46
- q.nest(0) do
47
- q.text(name)
48
- q.text(": ")
49
- visit(value)
50
- end
51
- end
52
-
53
- def list(name, values)
54
- q.group do
55
- q.text(name)
56
- q.text(": [")
57
- q.indent do
58
- q.breakable("")
59
- q.seplist(values) { |value| visit(value) }
60
- end
61
- q.breakable("")
62
- q.text("]")
63
- end
64
- end
65
-
66
- def node(node, _type)
67
- items = []
68
- q.with_target(items) { yield }
69
-
70
- if items.empty?
71
- q.text(node.class.name)
72
- return
73
- end
74
-
75
- q.group do
76
- q.text(node.class.name)
77
- q.text("[")
78
- q.indent do
79
- q.breakable("")
80
- q.seplist(items) { |item| q.target << item }
81
- end
82
- q.breakable("")
83
- q.text("]")
84
- end
85
- end
86
-
87
- def pairs(name, values)
88
- q.group do
89
- q.text(name)
90
- q.text(": [")
91
- q.indent do
92
- q.breakable("")
93
- q.seplist(values) do |(key, value)|
94
- q.group do
95
- q.text("[")
96
- q.indent do
97
- q.breakable("")
98
- visit(key)
99
- q.text(",")
100
- q.breakable
101
- visit(value || nil)
102
- end
103
- q.breakable("")
104
- q.text("]")
105
- end
106
- end
107
- end
108
- q.breakable("")
109
- q.text("]")
110
- end
111
- end
112
-
113
- def text(name, value)
114
- q.nest(0) do
115
- q.text(name)
116
- q.text(": ")
117
- value.pretty_print(q)
118
- end
119
- end
120
- end
121
- end
122
- end
@@ -1,85 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- class Visitor
5
- # This visitor pretty-prints the AST into an equivalent s-expression.
6
- class PrettyPrintVisitor < FieldVisitor
7
- attr_reader :q
8
-
9
- def initialize(q)
10
- @q = q
11
- end
12
-
13
- # This is here because we need to make sure the operator is cast to a
14
- # string before we print it out.
15
- def visit_binary(node)
16
- node(node, "binary") do
17
- field("left", node.left)
18
- text("operator", node.operator.to_s)
19
- field("right", node.right)
20
- comments(node)
21
- end
22
- end
23
-
24
- # This is here to make it a little nicer to look at labels since they
25
- # typically have their : at the end of the value.
26
- def visit_label(node)
27
- node(node, "label") do
28
- q.breakable
29
- q.text(":")
30
- q.text(node.value[0...-1])
31
- comments(node)
32
- end
33
- end
34
-
35
- private
36
-
37
- def comments(node)
38
- return if node.comments.empty?
39
-
40
- q.breakable
41
- q.group(2, "(", ")") do
42
- q.seplist(node.comments) { |comment| q.pp(comment) }
43
- end
44
- end
45
-
46
- def field(_name, value)
47
- q.breakable
48
- q.pp(value)
49
- end
50
-
51
- def list(_name, values)
52
- q.breakable
53
- q.group(2, "(", ")") { q.seplist(values) { |value| q.pp(value) } }
54
- end
55
-
56
- def node(_node, type)
57
- q.group(2, "(", ")") do
58
- q.text(type)
59
- yield
60
- end
61
- end
62
-
63
- def pairs(_name, values)
64
- q.group(2, "(", ")") do
65
- q.seplist(values) do |(key, value)|
66
- q.pp(key)
67
-
68
- if value
69
- q.text("=")
70
- q.group(2) do
71
- q.breakable("")
72
- q.pp(value)
73
- end
74
- end
75
- end
76
- end
77
- end
78
-
79
- def text(_name, value)
80
- q.breakable
81
- q.text(value)
82
- end
83
- end
84
- end
85
- end
@@ -1,140 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SyntaxTree
4
- # WithEnvironment is a module intended to be included in classes inheriting
5
- # from Visitor. The module overrides a few visit methods to automatically keep
6
- # track of local variables and arguments defined in the current environment.
7
- # Example usage:
8
- # class MyVisitor < Visitor
9
- # include WithEnvironment
10
- #
11
- # def visit_ident(node)
12
- # # Check if we're visiting an identifier for an argument, a local
13
- # variable or something else
14
- # local = current_environment.find_local(node)
15
- #
16
- # if local.type == :argument
17
- # # handle identifiers for arguments
18
- # elsif local.type == :variable
19
- # # handle identifiers for variables
20
- # else
21
- # # handle other identifiers, such as method names
22
- # end
23
- # end
24
- module WithEnvironment
25
- def current_environment
26
- @current_environment ||= Environment.new
27
- end
28
-
29
- def with_new_environment
30
- previous_environment = @current_environment
31
- @current_environment = Environment.new(previous_environment)
32
- yield
33
- ensure
34
- @current_environment = previous_environment
35
- end
36
-
37
- # Visits for nodes that create new environments, such as classes, modules
38
- # and method definitions
39
- def visit_class(node)
40
- with_new_environment { super }
41
- end
42
-
43
- def visit_module(node)
44
- with_new_environment { super }
45
- end
46
-
47
- # When we find a method invocation with a block, only the code that happens
48
- # inside of the block needs a fresh environment. The method invocation
49
- # itself happens in the same environment
50
- def visit_method_add_block(node)
51
- visit(node.call)
52
- with_new_environment { visit(node.block) }
53
- end
54
-
55
- def visit_def(node)
56
- with_new_environment { super }
57
- end
58
-
59
- # Visit for keeping track of local arguments, such as method and block
60
- # arguments
61
- def visit_params(node)
62
- add_argument_definitions(node.requireds)
63
-
64
- node.posts.each do |param|
65
- current_environment.add_local_definition(param, :argument)
66
- end
67
-
68
- node.keywords.each do |param|
69
- current_environment.add_local_definition(param.first, :argument)
70
- end
71
-
72
- node.optionals.each do |param|
73
- current_environment.add_local_definition(param.first, :argument)
74
- end
75
-
76
- super
77
- end
78
-
79
- def visit_rest_param(node)
80
- name = node.name
81
- current_environment.add_local_definition(name, :argument) if name
82
-
83
- super
84
- end
85
-
86
- def visit_kwrest_param(node)
87
- name = node.name
88
- current_environment.add_local_definition(name, :argument) if name
89
-
90
- super
91
- end
92
-
93
- def visit_blockarg(node)
94
- name = node.name
95
- current_environment.add_local_definition(name, :argument) if name
96
-
97
- super
98
- end
99
-
100
- # Visit for keeping track of local variable definitions
101
- def visit_var_field(node)
102
- value = node.value
103
-
104
- if value.is_a?(SyntaxTree::Ident)
105
- current_environment.add_local_definition(value, :variable)
106
- end
107
-
108
- super
109
- end
110
-
111
- alias visit_pinned_var_ref visit_var_field
112
-
113
- # Visits for keeping track of variable and argument usages
114
- def visit_var_ref(node)
115
- value = node.value
116
-
117
- if value.is_a?(SyntaxTree::Ident)
118
- definition = current_environment.find_local(value.value)
119
-
120
- if definition
121
- current_environment.add_local_usage(value, definition.type)
122
- end
123
- end
124
-
125
- super
126
- end
127
-
128
- private
129
-
130
- def add_argument_definitions(list)
131
- list.each do |param|
132
- if param.is_a?(SyntaxTree::MLHSParen)
133
- add_argument_definitions(param.contents.parts)
134
- else
135
- current_environment.add_local_definition(param, :argument)
136
- end
137
- end
138
- end
139
- end
140
- end