parser 2.3.0.2 → 2.3.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -78,7 +78,7 @@ module Parser
78
78
  !heredoc?)
79
79
 
80
80
  # Capture opening delimiter in percent-literals.
81
- @str_type << delimiter if @str_type.start_with?('%')
81
+ @str_type << delimiter if @str_type.start_with?('%'.freeze)
82
82
 
83
83
  clear_buffer
84
84
 
@@ -103,7 +103,7 @@ module Parser
103
103
  end
104
104
 
105
105
  def backslash_delimited?
106
- @end_delim == '\\'
106
+ @end_delim == '\\'.freeze
107
107
  end
108
108
 
109
109
  def type
@@ -116,7 +116,7 @@ module Parser
116
116
  if words? && character =~ /[ \t\v\r\f\n]/
117
117
  true
118
118
  else
119
- ['\\', @start_delim, @end_delim].include?(character)
119
+ ['\\'.freeze, @start_delim, @end_delim].include?(character)
120
120
  end
121
121
  end
122
122
 
@@ -135,8 +135,8 @@ module Parser
135
135
  extend_space(ts, ts)
136
136
  end
137
137
 
138
- if lookahead && lookahead[0] == ?: && lookahead[1] != ?: &&
139
- @label_allowed && @start_tok == :tSTRING_BEG
138
+ if lookahead && @label_allowed && lookahead[0] == ?: &&
139
+ lookahead[1] != ?: && @start_tok == :tSTRING_BEG
140
140
  # This is a quoted label.
141
141
  flush_string
142
142
  emit(:tLABEL_END, @end_delim, ts, te + 1)
@@ -40,8 +40,6 @@ module Parser
40
40
  )
41
41
  /x
42
42
 
43
- NEW_LINE = "\n".freeze
44
-
45
43
  ##
46
44
  # Try to recognize encoding of `string` as Ruby would, i.e. by looking for
47
45
  # magic encoding comment or UTF-8 BOM. `string` can be in any encoding.
@@ -58,7 +56,7 @@ module Parser
58
56
 
59
57
  if first_line =~ /\A\xef\xbb\xbf/ # BOM
60
58
  return Encoding::UTF_8
61
- elsif first_line[0, 2] == '#!'
59
+ elsif first_line[0, 2] == '#!'.freeze
62
60
  encoding_line = second_line
63
61
  else
64
62
  encoding_line = first_line
@@ -108,6 +106,10 @@ module Parser
108
106
 
109
107
  @lines = nil
110
108
  @line_begins = nil
109
+
110
+ # Cache for fast lookup
111
+ @line_for_position = {}
112
+ @col_for_position = {}
111
113
  end
112
114
 
113
115
  ##
@@ -175,7 +177,7 @@ module Parser
175
177
  raise ArgumentError, 'Source::Buffer is immutable'
176
178
  end
177
179
 
178
- @source = input.gsub("\r\n", NEW_LINE).freeze
180
+ @source = input.gsub("\r\n".freeze, "\n".freeze).freeze
179
181
  end
180
182
 
181
183
  ##
@@ -190,6 +192,34 @@ module Parser
190
192
  [ @first_line + line_no, position - line_begin ]
191
193
  end
192
194
 
195
+ ##
196
+ # Convert a character index into the source to a line number.
197
+ #
198
+ # @param [Integer] position
199
+ # @return [Integer] line
200
+ # @api private
201
+ #
202
+ def line_for_position(position)
203
+ @line_for_position[position] ||= begin
204
+ line_no, _ = line_for(position)
205
+ @first_line + line_no
206
+ end
207
+ end
208
+
209
+ ##
210
+ # Convert a character index into the source to a column number.
211
+ #
212
+ # @param [Integer] position
213
+ # @return [Integer] column
214
+ # @api private
215
+ #
216
+ def column_for_position(position)
217
+ @col_for_position[position] ||= begin
218
+ _, line_begin = line_for(position)
219
+ position - line_begin
220
+ end
221
+ end
222
+
193
223
  ##
194
224
  # Return an `Array` of source code lines.
195
225
  #
@@ -198,10 +228,10 @@ module Parser
198
228
  def source_lines
199
229
  @lines ||= begin
200
230
  lines = @source.lines.to_a
201
- lines << '' if @source.end_with?("\n")
231
+ lines << '' if @source.end_with?("\n".freeze)
202
232
 
203
233
  lines.each do |line|
204
- line.chomp!(NEW_LINE)
234
+ line.chomp!("\n".freeze)
205
235
  line.freeze
206
236
  end
207
237
 
@@ -253,14 +283,11 @@ module Parser
253
283
 
254
284
  def line_begins
255
285
  unless @line_begins
256
- @line_begins, index = [ [ 0, 0 ] ], 1
257
-
258
- @source.each_char do |char|
259
- if char == NEW_LINE
260
- @line_begins.unshift [ @line_begins.length, index ]
261
- end
286
+ @line_begins, index = [ [ 0, 0 ] ], 0
262
287
 
288
+ while index = @source.index("\n".freeze, index)
263
289
  index += 1
290
+ @line_begins.unshift [ @line_begins.length, index ]
264
291
  end
265
292
  end
266
293
 
@@ -120,12 +120,22 @@ module Parser
120
120
  def visit(node)
121
121
  process_leading_comments(node)
122
122
 
123
- node.children.each do |child|
124
- next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
125
- visit(child)
123
+ return unless @current_comment
124
+
125
+ # If the next comment is beyond the last line of this node, we don't
126
+ # need to iterate over its subnodes
127
+ # (Unless this node is a heredoc... there could be a comment in its body,
128
+ # inside an interpolation)
129
+ node_loc = node.location
130
+ if @current_comment.location.line <= node_loc.last_line ||
131
+ node_loc.is_a?(Map::Heredoc)
132
+ node.children.each do |child|
133
+ next unless child.is_a?(AST::Node) && child.loc && child.loc.expression
134
+ visit(child)
135
+ end
136
+
137
+ process_trailing_comments(node)
126
138
  end
127
-
128
- process_trailing_comments(node)
129
139
  end
130
140
 
131
141
  def process_leading_comments(node)
@@ -152,12 +162,8 @@ module Parser
152
162
  def current_comment_before?(node)
153
163
  return false if !@current_comment
154
164
  comment_loc = @current_comment.location.expression
155
-
156
- if node
157
- node_loc = node.location.expression
158
- return false if comment_loc.end_pos > node_loc.begin_pos
159
- end
160
- true
165
+ node_loc = node.location.expression
166
+ comment_loc.end_pos <= node_loc.begin_pos
161
167
  end
162
168
 
163
169
  def current_comment_before_end?(node)
@@ -34,6 +34,9 @@ module Parser
34
34
  if end_pos < begin_pos
35
35
  raise ArgumentError, 'Parser::Source::Range: end_pos must not be less than begin_pos'
36
36
  end
37
+ if source_buffer.nil?
38
+ raise ArgumentError, 'Parser::Source::Range: source_buffer must not be nil'
39
+ end
37
40
 
38
41
  @source_buffer = source_buffer
39
42
  @begin_pos, @end_pos = begin_pos, end_pos
@@ -74,9 +77,7 @@ module Parser
74
77
  # @return [Integer] line number of the beginning of this range.
75
78
  #
76
79
  def line
77
- line, _ = @source_buffer.decompose_position(@begin_pos)
78
-
79
- line
80
+ @source_buffer.line_for_position(@begin_pos)
80
81
  end
81
82
 
82
83
  alias_method :first_line, :line
@@ -85,27 +86,21 @@ module Parser
85
86
  # @return [Integer] zero-based column number of the beginning of this range.
86
87
  #
87
88
  def column
88
- _, column = @source_buffer.decompose_position(@begin_pos)
89
-
90
- column
89
+ @source_buffer.column_for_position(@begin_pos)
91
90
  end
92
91
 
93
92
  ##
94
93
  # @return [Integer] line number of the end of this range.
95
94
  #
96
95
  def last_line
97
- line, _ = @source_buffer.decompose_position(@end_pos)
98
-
99
- line
96
+ @source_buffer.line_for_position(@end_pos)
100
97
  end
101
98
 
102
99
  ##
103
100
  # @return [Integer] zero-based column number of the end of this range.
104
101
  #
105
102
  def last_column
106
- _, column = @source_buffer.decompose_position(@end_pos)
107
-
108
- column
103
+ @source_buffer.column_for_position(@end_pos)
109
104
  end
110
105
 
111
106
  ##
@@ -209,6 +204,21 @@ module Parser
209
204
  @begin_pos >= other.end_pos || other.begin_pos >= @end_pos
210
205
  end
211
206
 
207
+ ##
208
+ # @param [Range] other
209
+ # @return [Boolean] `true` if this range and `other` overlap
210
+ #
211
+ def overlaps?(other)
212
+ @begin_pos < other.end_pos && other.begin_pos < @end_pos
213
+ end
214
+
215
+ ##
216
+ # Checks if a range is empty; if it contains no characters
217
+ # @return [Boolean]
218
+ def empty?
219
+ @begin_pos == @end_pos
220
+ end
221
+
212
222
  ##
213
223
  # Compares ranges.
214
224
  # @return [Boolean]
@@ -5,7 +5,8 @@ module Parser
5
5
  # {Rewriter} performs the heavy lifting in the source rewriting process.
6
6
  # It schedules code updates to be performed in the correct order and
7
7
  # verifies that no two updates _clobber_ each other, that is, attempt to
8
- # modify the same part of code.
8
+ # modify the same section of code. (However, if two updates modify the
9
+ # same section in exactly the same way, they are merged.)
9
10
  #
10
11
  # If it is detected that one update clobbers another one, an `:error` and
11
12
  # a `:note` diagnostics describing both updates are generated and passed to
@@ -37,6 +38,7 @@ module Parser
37
38
  @source_buffer = source_buffer
38
39
  @queue = []
39
40
  @clobber = 0
41
+ @insertions = 0 # clobbered zero-length positions; index 0 is the far left
40
42
  end
41
43
 
42
44
  ##
@@ -47,7 +49,7 @@ module Parser
47
49
  # @raise [ClobberingError] when clobbering is detected
48
50
  #
49
51
  def remove(range)
50
- append Rewriter::Action.new(range, '')
52
+ append Rewriter::Action.new(range, ''.freeze)
51
53
  end
52
54
 
53
55
  ##
@@ -100,9 +102,7 @@ module Parser
100
102
  adjustment = 0
101
103
  source = @source_buffer.source.dup
102
104
 
103
- sorted_queue = @queue.sort_by.with_index do |action, index|
104
- [action.range.begin_pos, index]
105
- end
105
+ sorted_queue = @queue.sort_by { |action| action.range.begin_pos }
106
106
 
107
107
  sorted_queue.each do |action|
108
108
  begin_pos = action.range.begin_pos + adjustment
@@ -118,7 +118,7 @@ module Parser
118
118
 
119
119
  ##
120
120
  # Provides a protected block where a sequence of multiple rewrite actions
121
- # are handled atomic. If any of the action failed by clobbering,
121
+ # are handled atomically. If any of the actions failed by clobbering,
122
122
  # all the actions are rolled back.
123
123
  #
124
124
  # @example
@@ -144,110 +144,249 @@ module Parser
144
144
 
145
145
  @pending_queue = @queue.dup
146
146
  @pending_clobber = @clobber
147
+ @pending_insertions = @insertions
147
148
 
148
149
  yield
149
150
 
150
151
  @queue = @pending_queue
151
152
  @clobber = @pending_clobber
153
+ @insertions = @pending_insertions
152
154
 
153
155
  self
154
156
  ensure
155
157
  @pending_queue = nil
156
158
  @pending_clobber = nil
159
+ @pending_insertions = nil
157
160
  end
158
161
 
159
162
  private
160
163
 
164
+ # Schedule a code update. If it overlaps with another update, check
165
+ # whether they conflict, and raise a clobbering error if they do.
166
+ # (As a special case, zero-length ranges at the same position are
167
+ # considered to "overlap".) Otherwise, merge them.
168
+ #
169
+ # Updates which are adjacent to each other, but do not overlap, are also
170
+ # merged.
171
+ #
172
+ # RULES:
173
+ #
174
+ # - Insertion ("replacing" a zero-length range):
175
+ # - Two insertions at the same point conflict. This is true even
176
+ # if the earlier insertion has already been merged with an adjacent
177
+ # update, and even if they are both inserting the same text.
178
+ # - An insertion never conflicts with a replace or remove operation
179
+ # on its right or left side, which does not overlap it (in other
180
+ # words, which does not update BOTH its right and left sides).
181
+ # - An insertion always conflicts with a remove operation which spans
182
+ # both its sides.
183
+ # - An insertion conflicts with a replace operation which spans both its
184
+ # sides, unless the replacement text is longer than the replaced text
185
+ # by the size of the insertion (or more), and the portion of
186
+ # replacement text immediately after the insertion position is
187
+ # identical to the inserted text.
188
+ #
189
+ # - Removal operations never conflict with each other.
190
+ #
191
+ # - Replacement operations:
192
+ # - Take the portion of each replacement text which falls within:
193
+ # - The other operation's replaced region
194
+ # - The other operation's replacement text, if it extends past the
195
+ # end of its own replaced region (in other words, if the replacement
196
+ # text is longer than the text it replaces)
197
+ # - If and only if the taken texts are identical for both operations,
198
+ # they do not conflict.
199
+ #
161
200
  def append(action)
162
- if (clobber_actions = clobbered?(action.range))
163
- handle_clobber(action, clobber_actions)
201
+ range = action.range
202
+
203
+ # Is this an insertion?
204
+ if range.empty?
205
+ # Replacing nothing with... nothing?
206
+ return self if action.replacement.empty?
207
+
208
+ if (conflicting = clobbered_insertion?(range))
209
+ raise_clobber_error(action, [conflicting])
210
+ end
211
+
212
+ record_insertion(range)
213
+
214
+ if (adjacent = adjacent_updates?(range))
215
+ conflicting = adjacent.find do |a|
216
+ a.range.overlaps?(range) &&
217
+ !replace_compatible_with_insertion?(a, action)
218
+ end
219
+ raise_clobber_error(action, [conflicting]) if conflicting
220
+
221
+ merge_actions!(action, adjacent)
222
+ else
223
+ active_queue << action
224
+ end
164
225
  else
165
- clobber(action.range)
166
- active_queue << action
226
+ # It's a replace or remove operation.
227
+ if (insertions = adjacent_insertions?(range))
228
+ insertions.each do |insertion|
229
+ if range.overlaps?(insertion.range) &&
230
+ !replace_compatible_with_insertion?(action, insertion)
231
+ raise_clobber_error(action, [insertion])
232
+ else
233
+ action = merge_actions(action, [insertion])
234
+ active_queue.delete(insertion)
235
+ end
236
+ end
237
+ end
238
+
239
+ if (adjacent = adjacent_updates?(range))
240
+ if can_merge?(action, adjacent)
241
+ record_replace(range)
242
+ merge_actions!(action, adjacent)
243
+ else
244
+ raise_clobber_error(action, adjacent)
245
+ end
246
+ else
247
+ record_replace(range)
248
+ active_queue << action
249
+ end
167
250
  end
168
251
 
169
252
  self
170
253
  end
171
254
 
172
- def clobber(range)
173
- self.active_clobber = active_clobber | (2 ** range.size - 1) << range.begin_pos
255
+ def record_insertion(range)
256
+ self.active_insertions = active_insertions | (1 << range.begin_pos)
257
+ end
258
+
259
+ def record_replace(range)
260
+ self.active_clobber = active_clobber | clobbered_position_mask(range)
261
+ end
262
+
263
+ def clobbered_position_mask(range)
264
+ ((1 << range.size) - 1) << range.begin_pos
265
+ end
266
+
267
+ def adjacent_position_mask(range)
268
+ ((1 << (range.size + 2)) - 1) << (range.begin_pos - 1)
269
+ end
270
+
271
+ def adjacent_insertion_mask(range)
272
+ ((1 << (range.size + 1)) - 1) << range.begin_pos
174
273
  end
175
274
 
176
- def clobbered?(range)
177
- if active_clobber & ((2 ** range.size - 1) << range.begin_pos) != 0
178
- active_queue.select do |action|
179
- action.range.end_pos > range.begin_pos &&
180
- range.end_pos > action.range.begin_pos
275
+ def clobbered_insertion?(insertion)
276
+ insertion_pos = insertion.begin_pos
277
+ if active_insertions & (1 << insertion_pos) != 0
278
+ # The clobbered insertion may have already been merged with other
279
+ # updates, so it won't necessarily have the same begin_pos.
280
+ active_queue.find do |a|
281
+ a.range.begin_pos <= insertion_pos && insertion_pos <= a.range.end_pos
181
282
  end
182
283
  end
183
284
  end
184
285
 
185
- def handle_clobber(action, existing)
186
- if can_merge?(action, existing)
187
- merge_actions!(action, existing)
188
- else
189
- # cannot replace 3 characters with "foobar"
190
- diagnostic = Diagnostic.new(:error,
191
- :invalid_action,
192
- { :action => action },
193
- action.range)
194
- @diagnostics.process(diagnostic)
195
-
196
- # clobbered by: remove 3 characters
197
- diagnostic = Diagnostic.new(:note,
198
- :clobbered,
199
- { :action => existing[0] },
200
- existing[0].range)
201
- @diagnostics.process(diagnostic)
202
-
203
- raise ClobberingError, "Parser::Source::Rewriter detected clobbering"
286
+ def adjacent_insertions?(range)
287
+ # Just retrieve insertions which have not been merged with an adjacent
288
+ # remove or replace.
289
+ if active_insertions & adjacent_insertion_mask(range) != 0
290
+ result = active_queue.select do |a|
291
+ a.range.empty? && adjacent?(range, a.range)
292
+ end
293
+ result.empty? ? nil : result
294
+ end
295
+ end
296
+
297
+ def adjacent_updates?(range)
298
+ if active_clobber & adjacent_position_mask(range) != 0
299
+ active_queue.select { |a| adjacent?(range, a.range) }
204
300
  end
205
301
  end
206
302
 
303
+ def replace_compatible_with_insertion?(replace, insertion)
304
+ (replace.replacement.length - replace.range.size) >= insertion.range.size &&
305
+ (offset = insertion.range.begin_pos - replace.range.begin_pos) &&
306
+ replace.replacement[offset, insertion.replacement.length] == insertion.replacement
307
+ end
308
+
207
309
  def can_merge?(action, existing)
208
- existing.all? do |other|
209
- overlap = action.range.intersect(other.range)
210
- action_offset = overlap.begin_pos - action.range.begin_pos
211
- other_offset = overlap.begin_pos - other.range.begin_pos
310
+ # Compare 2 replace/remove operations (neither is an insertion)
311
+ range = action.range
212
312
 
213
- replacement1 = action.replacement[action_offset, overlap.size] || ''
214
- replacement2 = other.replacement[other_offset, overlap.size] || ''
313
+ existing.all? do |other|
314
+ overlap = range.intersect(other.range)
315
+ next true if overlap.nil? # adjacent, not overlapping
316
+
317
+ repl1_offset = overlap.begin_pos - range.begin_pos
318
+ repl2_offset = overlap.begin_pos - other.range.begin_pos
319
+ repl1_length = [other.range.length - repl2_offset,
320
+ other.replacement.length - repl2_offset].max
321
+ repl2_length = [range.length - repl1_offset,
322
+ action.replacement.length - repl1_offset].max
323
+
324
+ replacement1 = action.replacement[repl1_offset, repl1_length] || ''.freeze
325
+ replacement2 = other.replacement[repl2_offset, repl2_length] || ''.freeze
215
326
  replacement1 == replacement2
216
327
  end
217
328
  end
218
329
 
219
- def merge_actions!(action, existing)
220
- actions = existing.push(action).sort_by { |a| a.range.begin_pos }
221
- merged_begin = actions.map { |act| act.range.begin_pos }.min
222
- merged_end = actions.map { |act| act.range.end_pos }.max
223
- range = Source::Range.new(@source_buffer,
224
- merged_begin,
225
- merged_end)
226
- clobber(range)
227
-
228
- replacement = merge_replacements(actions)
229
- replace_actions(actions, Rewriter::Action.new(range, replacement))
330
+ def merge_actions(action, existing)
331
+ actions = existing.push(action).sort_by do |a|
332
+ [a.range.begin_pos, a.range.end_pos]
333
+ end
334
+ range = actions.first.range.join(actions.last.range)
335
+
336
+ Rewriter::Action.new(range, merge_replacements(actions))
230
337
  end
231
338
 
232
- def replace_actions(old, updated)
233
- old.each { |act| active_queue.delete(act) }
234
- active_queue << updated
339
+ def merge_actions!(action, existing)
340
+ new_action = merge_actions(action, existing)
341
+ active_queue.delete(action)
342
+ replace_actions(existing, new_action)
235
343
  end
236
344
 
237
345
  def merge_replacements(actions)
238
346
  # `actions` must be sorted by beginning position
239
347
  begin_pos = actions.first.range.begin_pos
240
348
  result = ''
349
+ prev_act = nil
241
350
 
242
351
  actions.each do |act|
243
- offset = result.size - act.range.begin_pos + begin_pos
244
- next if offset < 0 || offset >= act.replacement.size
245
- result << act.replacement[offset..-1]
352
+ if !prev_act || act.range.disjoint?(prev_act.range)
353
+ result << act.replacement
354
+ else
355
+ prev_end = [prev_act.range.begin_pos + prev_act.replacement.length,
356
+ prev_act.range.end_pos].max
357
+ offset = prev_end - act.range.begin_pos
358
+ result << act.replacement[offset..-1] if offset < act.replacement.size
359
+ end
360
+
361
+ prev_act = act
246
362
  end
247
363
 
248
364
  result
249
365
  end
250
366
 
367
+ def replace_actions(old, updated)
368
+ old.each { |act| active_queue.delete(act) }
369
+ active_queue << updated
370
+ end
371
+
372
+ def raise_clobber_error(action, existing)
373
+ # cannot replace 3 characters with "foobar"
374
+ diagnostic = Diagnostic.new(:error,
375
+ :invalid_action,
376
+ { :action => action },
377
+ action.range)
378
+ @diagnostics.process(diagnostic)
379
+
380
+ # clobbered by: remove 3 characters
381
+ diagnostic = Diagnostic.new(:note,
382
+ :clobbered,
383
+ { :action => existing[0] },
384
+ existing[0].range)
385
+ @diagnostics.process(diagnostic)
386
+
387
+ raise ClobberingError, "Parser::Source::Rewriter detected clobbering"
388
+ end
389
+
251
390
  def in_transaction?
252
391
  !@pending_queue.nil?
253
392
  end
@@ -260,6 +399,10 @@ module Parser
260
399
  @pending_clobber || @clobber
261
400
  end
262
401
 
402
+ def active_insertions
403
+ @pending_insertions || @insertions
404
+ end
405
+
263
406
  def active_clobber=(value)
264
407
  if @pending_clobber
265
408
  @pending_clobber = value
@@ -267,6 +410,18 @@ module Parser
267
410
  @clobber = value
268
411
  end
269
412
  end
413
+
414
+ def active_insertions=(value)
415
+ if @pending_insertions
416
+ @pending_insertions = value
417
+ else
418
+ @insertions = value
419
+ end
420
+ end
421
+
422
+ def adjacent?(range1, range2)
423
+ range1.begin_pos <= range2.end_pos && range2.begin_pos <= range1.end_pos
424
+ end
270
425
  end
271
426
 
272
427
  end