parser 2.7.0.4 → 2.7.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +18 -29
- data/CHANGELOG.md +53 -1
- data/README.md +6 -6
- data/Rakefile +2 -1
- data/doc/AST_FORMAT.md +54 -5
- data/lib/parser.rb +1 -0
- data/lib/parser/all.rb +1 -0
- data/lib/parser/ast/processor.rb +7 -0
- data/lib/parser/builders/default.rb +63 -14
- data/lib/parser/current.rb +13 -4
- data/lib/parser/diagnostic.rb +1 -1
- data/lib/parser/diagnostic/engine.rb +1 -2
- data/lib/parser/lexer.rl +7 -0
- data/lib/parser/messages.rb +15 -0
- data/lib/parser/meta.rb +2 -2
- data/lib/parser/ruby27.y +16 -5
- data/lib/parser/ruby28.y +3016 -0
- data/lib/parser/runner.rb +26 -2
- data/lib/parser/runner/ruby_rewrite.rb +2 -2
- data/lib/parser/source/buffer.rb +3 -1
- data/lib/parser/source/comment/associator.rb +14 -4
- data/lib/parser/source/map/endless_definition.rb +23 -0
- data/lib/parser/source/range.rb +16 -0
- data/lib/parser/source/tree_rewriter.rb +49 -12
- data/lib/parser/source/tree_rewriter/action.rb +96 -26
- data/lib/parser/tree_rewriter.rb +1 -2
- data/lib/parser/version.rb +1 -1
- data/parser.gemspec +2 -1
- data/test/helper.rb +25 -6
- data/test/parse_helper.rb +11 -17
- data/test/test_ast_processor.rb +32 -0
- data/test/test_base.rb +1 -1
- data/test/test_current.rb +2 -0
- data/test/test_diagnostic.rb +6 -7
- data/test/test_diagnostic_engine.rb +5 -8
- data/test/test_lexer.rb +17 -8
- data/test/test_meta.rb +12 -0
- data/test/test_parser.rb +260 -21
- data/test/test_runner_parse.rb +22 -1
- data/test/test_runner_rewrite.rb +1 -1
- data/test/test_source_buffer.rb +4 -1
- data/test/test_source_comment.rb +2 -2
- data/test/test_source_comment_associator.rb +47 -15
- data/test/test_source_map.rb +1 -2
- data/test/test_source_range.rb +29 -9
- data/test/test_source_rewriter.rb +4 -4
- data/test/test_source_rewriter_action.rb +2 -2
- data/test/test_source_tree_rewriter.rb +96 -6
- metadata +14 -7
data/lib/parser/runner.rb
CHANGED
@@ -14,9 +14,8 @@ module Parser
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def initialize
|
17
|
-
Parser::Builders::Default.modernize
|
18
|
-
|
19
17
|
@option_parser = OptionParser.new { |opts| setup_option_parsing(opts) }
|
18
|
+
@legacy = {}
|
20
19
|
@parser_class = nil
|
21
20
|
@parser = nil
|
22
21
|
@files = []
|
@@ -30,6 +29,7 @@ module Parser
|
|
30
29
|
|
31
30
|
def execute(options)
|
32
31
|
parse_options(options)
|
32
|
+
setup_builder_default
|
33
33
|
prepare_parser
|
34
34
|
|
35
35
|
process_all_input
|
@@ -37,6 +37,8 @@ module Parser
|
|
37
37
|
|
38
38
|
private
|
39
39
|
|
40
|
+
LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0].freeze
|
41
|
+
|
40
42
|
def runner_name
|
41
43
|
raise NotImplementedError, "implement #{self.class}##{__callee__}"
|
42
44
|
end
|
@@ -111,6 +113,11 @@ module Parser
|
|
111
113
|
@parser_class = Parser::Ruby27
|
112
114
|
end
|
113
115
|
|
116
|
+
opts.on '--28', 'Parse as Ruby 2.8 would' do
|
117
|
+
require 'parser/ruby28'
|
118
|
+
@parser_class = Parser::Ruby28
|
119
|
+
end
|
120
|
+
|
114
121
|
opts.on '--mac', 'Parse as MacRuby 0.12 would' do
|
115
122
|
require 'parser/macruby'
|
116
123
|
@parser_class = Parser::MacRuby
|
@@ -121,6 +128,17 @@ module Parser
|
|
121
128
|
@parser_class = Parser::RubyMotion
|
122
129
|
end
|
123
130
|
|
131
|
+
opts.on '--legacy', "Parse with all legacy modes" do
|
132
|
+
@legacy = Hash.new(true)
|
133
|
+
end
|
134
|
+
|
135
|
+
LEGACY_MODES.each do |mode|
|
136
|
+
opt_name = "--legacy-#{mode.to_s.gsub('_', '-')}"
|
137
|
+
opts.on opt_name, "Parse with legacy mode for emit_#{mode}" do
|
138
|
+
@legacy[mode] = true
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
124
142
|
opts.on '-w', '--warnings', 'Enable warnings' do |w|
|
125
143
|
@warnings = w
|
126
144
|
end
|
@@ -159,6 +177,12 @@ module Parser
|
|
159
177
|
end
|
160
178
|
end
|
161
179
|
|
180
|
+
def setup_builder_default
|
181
|
+
LEGACY_MODES.each do |mode|
|
182
|
+
Parser::Builders::Default.send(:"emit_#{mode}=", !@legacy[mode])
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
162
186
|
def prepare_parser
|
163
187
|
@parser = @parser_class.new
|
164
188
|
|
@@ -55,8 +55,8 @@ module Parser
|
|
55
55
|
new_source = rewriter.rewrite(buffer, ast)
|
56
56
|
|
57
57
|
new_buffer = Source::Buffer.new(initial_buffer.name +
|
58
|
-
'|after ' + rewriter_class.name
|
59
|
-
|
58
|
+
'|after ' + rewriter_class.name,
|
59
|
+
source: new_source)
|
60
60
|
|
61
61
|
@parser.reset
|
62
62
|
new_ast = @parser.parse(new_buffer)
|
data/lib/parser/source/buffer.rb
CHANGED
@@ -102,7 +102,7 @@ module Parser
|
|
102
102
|
end
|
103
103
|
end
|
104
104
|
|
105
|
-
def initialize(name, first_line = 1)
|
105
|
+
def initialize(name, first_line = 1, source: nil)
|
106
106
|
@name = name.to_s
|
107
107
|
@source = nil
|
108
108
|
@first_line = first_line
|
@@ -116,6 +116,8 @@ module Parser
|
|
116
116
|
# Cache for fast lookup
|
117
117
|
@line_for_position = {}
|
118
118
|
@column_for_position = {}
|
119
|
+
|
120
|
+
self.source = source if source
|
119
121
|
end
|
120
122
|
|
121
123
|
##
|
@@ -107,6 +107,19 @@ module Parser
|
|
107
107
|
|
108
108
|
private
|
109
109
|
|
110
|
+
POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post].freeze
|
111
|
+
def children_in_source_order(node)
|
112
|
+
if POSTFIX_TYPES.include?(node.type)
|
113
|
+
# All these types have either nodes with expressions, or `nil`
|
114
|
+
# so a compact will do, but they need to be sorted.
|
115
|
+
node.children.compact.sort_by { |child| child.loc.expression.begin_pos }
|
116
|
+
else
|
117
|
+
node.children.select do |child|
|
118
|
+
child.is_a?(AST::Node) && child.loc && child.loc.expression
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
110
123
|
def do_associate
|
111
124
|
@mapping = Hash.new { |h, k| h[k] = [] }
|
112
125
|
@comment_num = -1
|
@@ -131,10 +144,7 @@ module Parser
|
|
131
144
|
node_loc = node.location
|
132
145
|
if @current_comment.location.line <= node_loc.last_line ||
|
133
146
|
node_loc.is_a?(Map::Heredoc)
|
134
|
-
node.
|
135
|
-
next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
|
136
|
-
visit(child)
|
137
|
-
end
|
147
|
+
children_in_source_order(node).each { |child| visit(child) }
|
138
148
|
|
139
149
|
process_trailing_comments(node)
|
140
150
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Parser
|
4
|
+
module Source
|
5
|
+
|
6
|
+
class Map::EndlessDefinition < Map
|
7
|
+
attr_reader :keyword
|
8
|
+
attr_reader :operator
|
9
|
+
attr_reader :name
|
10
|
+
attr_reader :assignment
|
11
|
+
|
12
|
+
def initialize(keyword_l, operator_l, name_l, assignment_l, body_l)
|
13
|
+
@keyword = keyword_l
|
14
|
+
@operator = operator_l
|
15
|
+
@name = name_l
|
16
|
+
@assignment = assignment_l
|
17
|
+
|
18
|
+
super(@keyword.join(body_l))
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
data/lib/parser/source/range.rb
CHANGED
@@ -149,6 +149,13 @@ module Parser
|
|
149
149
|
(@begin_pos...@end_pos).to_a
|
150
150
|
end
|
151
151
|
|
152
|
+
##
|
153
|
+
# @return [Range] a Ruby range with the same `begin_pos` and `end_pos`
|
154
|
+
#
|
155
|
+
def to_range
|
156
|
+
self.begin_pos...self.end_pos
|
157
|
+
end
|
158
|
+
|
152
159
|
##
|
153
160
|
# Composes a GNU/Clang-style string representation of the beginning of this
|
154
161
|
# range.
|
@@ -298,6 +305,15 @@ module Parser
|
|
298
305
|
(@end_pos <=> other.end_pos)
|
299
306
|
end
|
300
307
|
|
308
|
+
alias_method :eql?, :==
|
309
|
+
|
310
|
+
##
|
311
|
+
# Support for Ranges be used in as Hash indices and in Sets.
|
312
|
+
#
|
313
|
+
def hash
|
314
|
+
[@source_buffer, @begin_pos, @end_pos].hash
|
315
|
+
end
|
316
|
+
|
301
317
|
##
|
302
318
|
# @return [String] a human-readable representation of this range.
|
303
319
|
#
|
@@ -117,6 +117,43 @@ module Parser
|
|
117
117
|
@action_root = TreeRewriter::Action.new(all_encompassing_range, @enforcer)
|
118
118
|
end
|
119
119
|
|
120
|
+
##
|
121
|
+
# Returns true iff no (non trivial) update has been recorded
|
122
|
+
#
|
123
|
+
# @return [Boolean]
|
124
|
+
#
|
125
|
+
def empty?
|
126
|
+
@action_root.empty?
|
127
|
+
end
|
128
|
+
|
129
|
+
##
|
130
|
+
# Merges the updates of argument with the receiver.
|
131
|
+
# Policies of the receiver are used.
|
132
|
+
#
|
133
|
+
# @param [Rewriter] with
|
134
|
+
# @return [Rewriter] self
|
135
|
+
# @raise [ClobberingError] when clobbering is detected
|
136
|
+
#
|
137
|
+
def merge!(with)
|
138
|
+
raise 'TreeRewriter are not for the same source_buffer' unless
|
139
|
+
source_buffer == with.source_buffer
|
140
|
+
|
141
|
+
@action_root = @action_root.combine(with.action_root)
|
142
|
+
self
|
143
|
+
end
|
144
|
+
|
145
|
+
##
|
146
|
+
# Returns a new rewriter that consists of the updates of the received
|
147
|
+
# and the given argument. Policies of the receiver are used.
|
148
|
+
#
|
149
|
+
# @param [Rewriter] with
|
150
|
+
# @return [Rewriter] merge of receiver and argument
|
151
|
+
# @raise [ClobberingError] when clobbering is detected
|
152
|
+
#
|
153
|
+
def merge(with)
|
154
|
+
dup.merge!(with)
|
155
|
+
end
|
156
|
+
|
120
157
|
##
|
121
158
|
# Replaces the code of the source range `range` with `content`.
|
122
159
|
#
|
@@ -185,28 +222,24 @@ module Parser
|
|
185
222
|
# @return [String]
|
186
223
|
#
|
187
224
|
def process
|
188
|
-
source = @source_buffer.source
|
189
|
-
adjustment = 0
|
225
|
+
source = @source_buffer.source
|
190
226
|
|
227
|
+
chunks = []
|
228
|
+
last_end = 0
|
191
229
|
@action_root.ordered_replacements.each do |range, replacement|
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
source[begin_pos...end_pos] = replacement
|
196
|
-
|
197
|
-
adjustment += replacement.length - range.length
|
230
|
+
chunks << source[last_end...range.begin_pos] << replacement
|
231
|
+
last_end = range.end_pos
|
198
232
|
end
|
199
|
-
|
200
|
-
|
233
|
+
chunks << source[last_end...source.length]
|
234
|
+
chunks.join
|
201
235
|
end
|
202
236
|
|
203
237
|
##
|
204
238
|
# Provides a protected block where a sequence of multiple rewrite actions
|
205
239
|
# are handled atomically. If any of the actions failed by clobbering,
|
206
|
-
# all the actions are rolled back.
|
240
|
+
# all the actions are rolled back. Transactions can be nested.
|
207
241
|
#
|
208
242
|
# @raise [RuntimeError] when no block is passed
|
209
|
-
# @raise [RuntimeError] when already in a transaction
|
210
243
|
#
|
211
244
|
def transaction
|
212
245
|
unless block_given?
|
@@ -256,6 +289,10 @@ module Parser
|
|
256
289
|
|
257
290
|
extend Deprecation
|
258
291
|
|
292
|
+
protected
|
293
|
+
|
294
|
+
attr_reader :action_root
|
295
|
+
|
259
296
|
private
|
260
297
|
|
261
298
|
ACTIONS = %i[accept warn raise].freeze
|
@@ -7,7 +7,7 @@ module Parser
|
|
7
7
|
#
|
8
8
|
# Actions are arranged in a tree and get combined so that:
|
9
9
|
# children are strictly contained by their parent
|
10
|
-
# sibblings all disjoint from one another
|
10
|
+
# sibblings all disjoint from one another and ordered
|
11
11
|
# only actions with replacement==nil may have children
|
12
12
|
#
|
13
13
|
class TreeRewriter::Action
|
@@ -25,17 +25,23 @@ module Parser
|
|
25
25
|
freeze
|
26
26
|
end
|
27
27
|
|
28
|
-
# Assumes action.children.empty?
|
29
28
|
def combine(action)
|
30
|
-
return self
|
29
|
+
return self if action.empty? # Ignore empty action
|
31
30
|
do_combine(action)
|
32
31
|
end
|
33
32
|
|
33
|
+
def empty?
|
34
|
+
@insert_before.empty? &&
|
35
|
+
@insert_after.empty? &&
|
36
|
+
@children.empty? &&
|
37
|
+
(@replacement == nil || (@replacement.empty? && @range.empty?))
|
38
|
+
end
|
39
|
+
|
34
40
|
def ordered_replacements
|
35
41
|
reps = []
|
36
42
|
reps << [@range.begin, @insert_before] unless @insert_before.empty?
|
37
43
|
reps << [@range, @replacement] if @replacement
|
38
|
-
reps.concat(@children.
|
44
|
+
reps.concat(@children.flat_map(&:ordered_replacements))
|
39
45
|
reps << [@range.end, @insert_after] unless @insert_after.empty?
|
40
46
|
reps
|
41
47
|
end
|
@@ -46,9 +52,11 @@ module Parser
|
|
46
52
|
|
47
53
|
protected
|
48
54
|
|
49
|
-
|
55
|
+
attr_reader :children
|
56
|
+
|
57
|
+
def with(range: @range, enforcer: @enforcer, children: @children, insert_before: @insert_before, replacement: @replacement, insert_after: @insert_after)
|
50
58
|
children = swallow(children) if replacement
|
51
|
-
self.class.new(range,
|
59
|
+
self.class.new(range, enforcer, children: children, insert_before: insert_before, replacement: replacement, insert_after: insert_after)
|
52
60
|
end
|
53
61
|
|
54
62
|
# Assumes range.contains?(action.range) && action.children.empty?
|
@@ -61,19 +69,27 @@ module Parser
|
|
61
69
|
end
|
62
70
|
|
63
71
|
def place_in_hierarchy(action)
|
64
|
-
family =
|
72
|
+
family = analyse_hierarchy(action)
|
65
73
|
|
66
74
|
if family[:fusible]
|
67
|
-
fuse_deletions(action, family[:fusible], [*family[:
|
75
|
+
fuse_deletions(action, family[:fusible], [*family[:sibbling_left], *family[:child], *family[:sibbling_right]])
|
68
76
|
else
|
69
77
|
extra_sibbling = if family[:parent] # action should be a descendant of one of the children
|
70
|
-
family[:parent]
|
78
|
+
family[:parent].do_combine(action)
|
71
79
|
elsif family[:child] # or it should become the parent of some of the children,
|
72
|
-
action.with(children: family[:child])
|
80
|
+
action.with(children: family[:child], enforcer: @enforcer)
|
81
|
+
.combine_children(action.children)
|
73
82
|
else # or else it should become an additional child
|
74
83
|
action
|
75
84
|
end
|
76
|
-
with(children: [*family[:
|
85
|
+
with(children: [*family[:sibbling_left], extra_sibbling, *family[:sibbling_right]])
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Assumes `more_children` all contained within `@range`
|
90
|
+
def combine_children(more_children)
|
91
|
+
more_children.inject(self) do |parent, new_child|
|
92
|
+
parent.place_in_hierarchy(new_child)
|
77
93
|
end
|
78
94
|
end
|
79
95
|
|
@@ -84,22 +100,76 @@ module Parser
|
|
84
100
|
without_fusible.do_combine(fused_deletion)
|
85
101
|
end
|
86
102
|
|
87
|
-
#
|
88
|
-
#
|
89
|
-
#
|
90
|
-
def
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
103
|
+
# Similar to @children.bsearch_index || size
|
104
|
+
# except allows for a starting point
|
105
|
+
# and `bsearch_index` is only Ruby 2.3+
|
106
|
+
def bsearch_child_index(from = 0)
|
107
|
+
size = @children.size
|
108
|
+
(from...size).bsearch { |i| yield @children[i] } || size
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns the children in a hierarchy with respect to `action`:
|
112
|
+
# :sibbling_left, sibbling_right (for those that are disjoint from `action`)
|
113
|
+
# :parent (in case one of our children contains `action`)
|
114
|
+
# :child (in case `action` strictly contains some of our children)
|
115
|
+
# :fusible (in case `action` overlaps some children but they can be fused in one deletion)
|
116
|
+
# or raises a `CloberingError`
|
117
|
+
# In case a child has equal range to `action`, it is returned as `:parent`
|
118
|
+
# Reminder: an empty range 1...1 is considered disjoint from 1...10
|
119
|
+
def analyse_hierarchy(action)
|
120
|
+
r = action.range
|
121
|
+
# left_index is the index of the first child that isn't completely to the left of action
|
122
|
+
left_index = bsearch_child_index { |child| child.range.end_pos > r.begin_pos }
|
123
|
+
# right_index is the index of the first child that is completely on the right of action
|
124
|
+
start = left_index == 0 ? 0 : left_index - 1 # See "corner case" below for reason of -1
|
125
|
+
right_index = bsearch_child_index(start) { |child| child.range.begin_pos >= r.end_pos }
|
126
|
+
center = right_index - left_index
|
127
|
+
case center
|
128
|
+
when 0
|
129
|
+
# All children are disjoint from action, nothing else to do
|
130
|
+
when -1
|
131
|
+
# Corner case: if a child has empty range == action's range
|
132
|
+
# then it will appear to be both disjoint and to the left of action,
|
133
|
+
# as well as disjoint and to the right of action.
|
134
|
+
# Since ranges are equal, we return it as parent
|
135
|
+
left_index -= 1 # Fix indices, as otherwise this child would be
|
136
|
+
right_index += 1 # considered as a sibbling (both left and right!)
|
137
|
+
parent = @children[left_index]
|
100
138
|
else
|
101
|
-
@
|
139
|
+
overlap_left = @children[left_index].range.begin_pos <=> r.begin_pos
|
140
|
+
overlap_right = @children[right_index-1].range.end_pos <=> r.end_pos
|
141
|
+
|
142
|
+
# For one child to be the parent of action, we must have:
|
143
|
+
if center == 1 && overlap_left <= 0 && overlap_right >= 0
|
144
|
+
parent = @children[left_index]
|
145
|
+
else
|
146
|
+
# Otherwise consider all non disjoint elements (center) to be contained...
|
147
|
+
contained = @children[left_index...right_index]
|
148
|
+
fusible = check_fusible(action,
|
149
|
+
(contained.shift if overlap_left < 0), # ... but check first and last one
|
150
|
+
(contained.pop if overlap_right > 0) # ... for overlaps
|
151
|
+
)
|
152
|
+
end
|
102
153
|
end
|
154
|
+
|
155
|
+
{
|
156
|
+
parent: parent,
|
157
|
+
sibbling_left: @children[0...left_index],
|
158
|
+
sibbling_right: @children[right_index...@children.size],
|
159
|
+
fusible: fusible,
|
160
|
+
child: contained,
|
161
|
+
}
|
162
|
+
end
|
163
|
+
|
164
|
+
# @param [Array(Action | nil)] fusible
|
165
|
+
def check_fusible(action, *fusible)
|
166
|
+
fusible.compact!
|
167
|
+
return if fusible.empty?
|
168
|
+
fusible.each do |child|
|
169
|
+
kind = action.insertion? || child.insertion? ? :crossing_insertions : :crossing_deletions
|
170
|
+
@enforcer.call(kind) { {range: action.range, conflict: child.range} }
|
171
|
+
end
|
172
|
+
fusible
|
103
173
|
end
|
104
174
|
|
105
175
|
# Assumes action.range == range && action.children.empty?
|
@@ -109,7 +179,7 @@ module Parser
|
|
109
179
|
insert_before: "#{action.insert_before}#{insert_before}",
|
110
180
|
replacement: action.replacement || @replacement,
|
111
181
|
insert_after: "#{insert_after}#{action.insert_after}",
|
112
|
-
)
|
182
|
+
).combine_children(action.children)
|
113
183
|
end
|
114
184
|
|
115
185
|
def call_enforcer_for_merge(action)
|