parser 2.7.1.3 → 3.0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -1
  3. data/lib/parser/all.rb +1 -1
  4. data/lib/parser/ast/processor.rb +5 -7
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +225 -29
  7. data/lib/parser/context.rb +5 -0
  8. data/lib/parser/current.rb +11 -11
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/lexer.rb +23780 -0
  11. data/lib/parser/macruby.rb +6149 -0
  12. data/lib/parser/max_numparam_stack.rb +13 -5
  13. data/lib/parser/messages.rb +3 -0
  14. data/lib/parser/meta.rb +8 -7
  15. data/lib/parser/ruby18.rb +5667 -0
  16. data/lib/parser/ruby19.rb +6092 -0
  17. data/lib/parser/ruby20.rb +6527 -0
  18. data/lib/parser/ruby21.rb +6578 -0
  19. data/lib/parser/ruby22.rb +6613 -0
  20. data/lib/parser/ruby23.rb +6624 -0
  21. data/lib/parser/ruby24.rb +6694 -0
  22. data/lib/parser/ruby25.rb +6662 -0
  23. data/lib/parser/ruby26.rb +6676 -0
  24. data/lib/parser/ruby27.rb +7862 -0
  25. data/lib/parser/ruby28.rb +8047 -0
  26. data/lib/parser/ruby30.rb +8060 -0
  27. data/lib/parser/rubymotion.rb +6086 -0
  28. data/lib/parser/runner.rb +4 -4
  29. data/lib/parser/source/buffer.rb +50 -27
  30. data/lib/parser/source/comment.rb +1 -1
  31. data/lib/parser/source/comment/associator.rb +1 -1
  32. data/lib/parser/source/map/{endless_definition.rb → method_definition.rb} +5 -3
  33. data/lib/parser/source/range.rb +3 -3
  34. data/lib/parser/source/tree_rewriter.rb +94 -1
  35. data/lib/parser/source/tree_rewriter/action.rb +39 -0
  36. data/lib/parser/static_environment.rb +4 -0
  37. data/lib/parser/variables_stack.rb +4 -0
  38. data/lib/parser/version.rb +1 -1
  39. data/parser.gemspec +2 -18
  40. metadata +13 -102
  41. data/.gitignore +0 -34
  42. data/.travis.yml +0 -40
  43. data/.yardopts +0 -21
  44. data/CHANGELOG.md +0 -1101
  45. data/CONTRIBUTING.md +0 -17
  46. data/Gemfile +0 -10
  47. data/README.md +0 -308
  48. data/Rakefile +0 -167
  49. data/ci/run_rubocop_specs +0 -14
  50. data/doc/AST_FORMAT.md +0 -2229
  51. data/doc/CUSTOMIZATION.md +0 -37
  52. data/doc/INTERNALS.md +0 -21
  53. data/doc/css/.gitkeep +0 -0
  54. data/doc/css/common.css +0 -68
  55. data/lib/parser/lexer.rl +0 -2543
  56. data/lib/parser/macruby.y +0 -2198
  57. data/lib/parser/ruby18.y +0 -1934
  58. data/lib/parser/ruby19.y +0 -2175
  59. data/lib/parser/ruby20.y +0 -2353
  60. data/lib/parser/ruby21.y +0 -2357
  61. data/lib/parser/ruby22.y +0 -2364
  62. data/lib/parser/ruby23.y +0 -2370
  63. data/lib/parser/ruby24.y +0 -2408
  64. data/lib/parser/ruby25.y +0 -2405
  65. data/lib/parser/ruby26.y +0 -2413
  66. data/lib/parser/ruby27.y +0 -2941
  67. data/lib/parser/ruby28.y +0 -3016
  68. data/lib/parser/rubymotion.y +0 -2182
  69. data/test/bug_163/fixtures/input.rb +0 -5
  70. data/test/bug_163/fixtures/output.rb +0 -5
  71. data/test/bug_163/rewriter.rb +0 -20
  72. data/test/helper.rb +0 -79
  73. data/test/parse_helper.rb +0 -313
  74. data/test/racc_coverage_helper.rb +0 -133
  75. data/test/test_ast_processor.rb +0 -32
  76. data/test/test_base.rb +0 -31
  77. data/test/test_current.rb +0 -31
  78. data/test/test_diagnostic.rb +0 -95
  79. data/test/test_diagnostic_engine.rb +0 -59
  80. data/test/test_encoding.rb +0 -99
  81. data/test/test_lexer.rb +0 -3617
  82. data/test/test_lexer_stack_state.rb +0 -78
  83. data/test/test_meta.rb +0 -12
  84. data/test/test_parse_helper.rb +0 -80
  85. data/test/test_parser.rb +0 -9596
  86. data/test/test_runner_parse.rb +0 -56
  87. data/test/test_runner_rewrite.rb +0 -47
  88. data/test/test_source_buffer.rb +0 -165
  89. data/test/test_source_comment.rb +0 -36
  90. data/test/test_source_comment_associator.rb +0 -399
  91. data/test/test_source_map.rb +0 -14
  92. data/test/test_source_range.rb +0 -192
  93. data/test/test_source_rewriter.rb +0 -541
  94. data/test/test_source_rewriter_action.rb +0 -46
  95. data/test/test_source_tree_rewriter.rb +0 -263
  96. data/test/test_static_environment.rb +0 -45
  97. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  98. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  99. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
data/lib/parser/runner.rb CHANGED
@@ -37,7 +37,7 @@ module Parser
37
37
 
38
38
  private
39
39
 
40
- LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0].freeze
40
+ LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0 forward_arg kwargs match_pattern].freeze
41
41
 
42
42
  def runner_name
43
43
  raise NotImplementedError, "implement #{self.class}##{__callee__}"
@@ -113,9 +113,9 @@ module Parser
113
113
  @parser_class = Parser::Ruby27
114
114
  end
115
115
 
116
- opts.on '--28', 'Parse as Ruby 2.8 would' do
117
- require 'parser/ruby28'
118
- @parser_class = Parser::Ruby28
116
+ opts.on '--30', 'Parse as Ruby 3.0 would' do
117
+ require 'parser/ruby30'
118
+ @parser_class = Parser::Ruby30
119
119
  end
120
120
 
121
121
  opts.on '--mac', 'Parse as MacRuby 0.12 would' do
@@ -114,8 +114,7 @@ module Parser
114
114
  @slice_source = nil
115
115
 
116
116
  # Cache for fast lookup
117
- @line_for_position = {}
118
- @column_for_position = {}
117
+ @line_index_for_position = {}
119
118
 
120
119
  self.source = source if source
121
120
  end
@@ -207,9 +206,10 @@ module Parser
207
206
  # @return [[Integer, Integer]] `[line, column]`
208
207
  #
209
208
  def decompose_position(position)
210
- line_no, line_begin = line_for(position)
209
+ line_index = line_index_for_position(position)
210
+ line_begin = line_begins[line_index]
211
211
 
212
- [ @first_line + line_no, position - line_begin ]
212
+ [ @first_line + line_index , position - line_begin ]
213
213
  end
214
214
 
215
215
  ##
@@ -220,10 +220,7 @@ module Parser
220
220
  # @api private
221
221
  #
222
222
  def line_for_position(position)
223
- @line_for_position[position] ||= begin
224
- line_no, _ = line_for(position)
225
- @first_line + line_no
226
- end
223
+ line_index_for_position(position) + @first_line
227
224
  end
228
225
 
229
226
  ##
@@ -234,10 +231,8 @@ module Parser
234
231
  # @api private
235
232
  #
236
233
  def column_for_position(position)
237
- @column_for_position[position] ||= begin
238
- _, line_begin = line_for(position)
239
- position - line_begin
240
- end
234
+ line_index = line_index_for_position(position)
235
+ position - line_begins[line_index]
241
236
  end
242
237
 
243
238
  ##
@@ -278,15 +273,13 @@ module Parser
278
273
  # @raise [IndexError] if `lineno` is out of bounds
279
274
  #
280
275
  def line_range(lineno)
281
- index = lineno - @first_line + 1
282
- if index <= 0 || index > line_begins.size
276
+ index = lineno - @first_line
277
+ if index < 0 || index + 1 >= line_begins.size
283
278
  raise IndexError, 'Parser::Source::Buffer: range for line ' \
284
279
  "#{lineno} requested, valid line numbers are #{@first_line}.." \
285
- "#{@first_line + line_begins.size - 1}"
286
- elsif index == line_begins.size
287
- Range.new(self, line_begins[-index][1], @source.size)
280
+ "#{@first_line + line_begins.size - 2}"
288
281
  else
289
- Range.new(self, line_begins[-index][1], line_begins[-index - 1][1] - 1)
282
+ Range.new(self, line_begins[index], line_begins[index + 1] - 1)
290
283
  end
291
284
  end
292
285
 
@@ -303,27 +296,57 @@ module Parser
303
296
  # @return [Integer]
304
297
  #
305
298
  def last_line
306
- line_begins.size + @first_line - 1
299
+ line_begins.size + @first_line - 2
300
+ end
301
+
302
+ # :nodoc:
303
+ def freeze
304
+ source_lines; line_begins; source_range # build cache
305
+ super
306
+ end
307
+
308
+ # :nodoc:
309
+ def inspect
310
+ "#<#{self.class} #{name}>"
307
311
  end
308
312
 
309
313
  private
310
314
 
315
+ # @returns [0, line_begin_of_line_1, ..., source.size + 1]
311
316
  def line_begins
312
- unless @line_begins
313
- @line_begins, index = [ [ 0, 0 ] ], 0
314
-
317
+ @line_begins ||= begin
318
+ begins = [0]
319
+ index = 0
315
320
  while index = @source.index("\n".freeze, index)
316
321
  index += 1
317
- @line_begins.unshift [ @line_begins.length, index ]
322
+ begins << index
318
323
  end
324
+ begins << @source.size + 1
325
+ begins
319
326
  end
327
+ end
320
328
 
321
- @line_begins
329
+ # @returns 0-based line index of position
330
+ def line_index_for_position(position)
331
+ @line_index_for_position[position] || begin
332
+ index = bsearch(line_begins, position) - 1
333
+ @line_index_for_position[position] = index unless @line_index_for_position.frozen?
334
+ index
335
+ end
322
336
  end
323
337
 
324
- def line_for(position)
325
- line_begins.bsearch do |line, line_begin|
326
- line_begin <= position
338
+ if Array.method_defined?(:bsearch_index) # RUBY_VERSION >= 2.3
339
+ def bsearch(line_begins, position)
340
+ line_begins.bsearch_index do |line_begin|
341
+ position < line_begin
342
+ end || line_begins.size - 1 # || only for out of bound values
343
+ end
344
+ else
345
+ def bsearch(line_begins, position)
346
+ @line_range ||= 0...line_begins.size
347
+ @line_range.bsearch do |i|
348
+ position < line_begins[i]
349
+ end || line_begins.size - 1 # || only for out of bound values
327
350
  end
328
351
  end
329
352
  end
@@ -10,7 +10,7 @@ module Parser
10
10
  # @return [String]
11
11
  #
12
12
  # @!attribute [r] location
13
- # @return [Parser::Source::Map]
13
+ # @return [Parser::Source::Range]
14
14
  #
15
15
  # @api public
16
16
  #
@@ -107,7 +107,7 @@ module Parser
107
107
 
108
108
  private
109
109
 
110
- POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post].freeze
110
+ POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post, :masgn].freeze
111
111
  def children_in_source_order(node)
112
112
  if POSTFIX_TYPES.include?(node.type)
113
113
  # All these types have either nodes with expressions, or `nil`
@@ -3,19 +3,21 @@
3
3
  module Parser
4
4
  module Source
5
5
 
6
- class Map::EndlessDefinition < Map
6
+ class Map::MethodDefinition < Map
7
7
  attr_reader :keyword
8
8
  attr_reader :operator
9
9
  attr_reader :name
10
+ attr_reader :end
10
11
  attr_reader :assignment
11
12
 
12
- def initialize(keyword_l, operator_l, name_l, assignment_l, body_l)
13
+ def initialize(keyword_l, operator_l, name_l, end_l, assignment_l, body_l)
13
14
  @keyword = keyword_l
14
15
  @operator = operator_l
15
16
  @name = name_l
17
+ @end = end_l
16
18
  @assignment = assignment_l
17
19
 
18
- super(@keyword.join(body_l))
20
+ super(@keyword.join(end_l || body_l))
19
21
  end
20
22
  end
21
23
 
@@ -13,7 +13,7 @@ module Parser
13
13
  # ^^
14
14
  #
15
15
  # @!attribute [r] source_buffer
16
- # @return [Parser::Diagnostic::Engine]
16
+ # @return [Parser::Source::Buffer]
17
17
  #
18
18
  # @!attribute [r] begin_pos
19
19
  # @return [Integer] index of the first character in the range
@@ -112,11 +112,11 @@ module Parser
112
112
  # @raise RangeError
113
113
  #
114
114
  def column_range
115
- if self.begin.line != self.end.line
115
+ if line != last_line
116
116
  raise RangeError, "#{self.inspect} spans more than one line"
117
117
  end
118
118
 
119
- self.begin.column...self.end.column
119
+ column...last_column
120
120
  end
121
121
 
122
122
  ##
@@ -129,6 +129,8 @@ module Parser
129
129
  ##
130
130
  # Merges the updates of argument with the receiver.
131
131
  # Policies of the receiver are used.
132
+ # This action is atomic in that it won't change the receiver
133
+ # unless it succeeds.
132
134
  #
133
135
  # @param [Rewriter] with
134
136
  # @return [Rewriter] self
@@ -154,6 +156,32 @@ module Parser
154
156
  dup.merge!(with)
155
157
  end
156
158
 
159
+ ##
160
+ # For special cases where one needs to merge a rewriter attached to a different source_buffer
161
+ # or that needs to be offset. Policies of the receiver are used.
162
+ #
163
+ # @param [TreeRewriter] rewriter from different source_buffer
164
+ # @param [Integer] offset
165
+ # @return [Rewriter] self
166
+ # @raise [IndexError] if action ranges (once offset) don't fit the current buffer
167
+ #
168
+ def import!(foreign_rewriter, offset: 0)
169
+ return self if foreign_rewriter.empty?
170
+
171
+ contracted = foreign_rewriter.action_root.contract
172
+ merge_effective_range = ::Parser::Source::Range.new(
173
+ @source_buffer,
174
+ contracted.range.begin_pos + offset,
175
+ contracted.range.end_pos + offset,
176
+ )
177
+ check_range_validity(merge_effective_range)
178
+
179
+ merge_with = contracted.moved(@source_buffer, offset)
180
+
181
+ @action_root = @action_root.combine(merge_with)
182
+ self
183
+ end
184
+
157
185
  ##
158
186
  # Replaces the code of the source range `range` with `content`.
159
187
  #
@@ -234,6 +262,44 @@ module Parser
234
262
  chunks.join
235
263
  end
236
264
 
265
+ ##
266
+ # Returns a representation of the rewriter as an ordered list of replacements.
267
+ #
268
+ # rewriter.as_replacements # => [ [1...1, '('],
269
+ # [2...4, 'foo'],
270
+ # [5...6, ''],
271
+ # [6...6, '!'],
272
+ # [10...10, ')'],
273
+ # ]
274
+ #
275
+ # This representation is sufficient to recreate the result of `process` but it is
276
+ # not sufficient to recreate completely the rewriter for further merging/actions.
277
+ # See `as_nested_actions`
278
+ #
279
+ # @return [Array<Range, String>] an ordered list of pairs of range & replacement
280
+ #
281
+ def as_replacements
282
+ @action_root.ordered_replacements
283
+ end
284
+
285
+ ##
286
+ # Returns a representation of the rewriter as nested insertions (:wrap) and replacements.
287
+ #
288
+ # rewriter.as_actions # =>[ [:wrap, 1...10, '(', ')'],
289
+ # [:wrap, 2...6, '', '!'], # aka "insert_after"
290
+ # [:replace, 2...4, 'foo'],
291
+ # [:replace, 5...6, ''], # aka "removal"
292
+ # ],
293
+ #
294
+ # Contrary to `as_replacements`, this representation is sufficient to recreate exactly
295
+ # the rewriter.
296
+ #
297
+ # @return [Array<(Symbol, Range, String{, String})>]
298
+ #
299
+ def as_nested_actions
300
+ @action_root.nested_actions
301
+ end
302
+
237
303
  ##
238
304
  # Provides a protected block where a sequence of multiple rewrite actions
239
305
  # are handled atomically. If any of the actions failed by clobbering,
@@ -264,6 +330,11 @@ module Parser
264
330
  @in_transaction
265
331
  end
266
332
 
333
+ # :nodoc:
334
+ def inspect
335
+ "#<#{self.class} #{source_buffer.name}: #{action_summary}>"
336
+ end
337
+
267
338
  ##
268
339
  # @api private
269
340
  # @deprecated Use insert_after or wrap
@@ -295,6 +366,28 @@ module Parser
295
366
 
296
367
  private
297
368
 
369
+ def action_summary
370
+ replacements = as_replacements
371
+ case replacements.size
372
+ when 0 then return 'empty'
373
+ when 1..3 then #ok
374
+ else
375
+ replacements = replacements.first(3)
376
+ suffix = '…'
377
+ end
378
+ parts = replacements.map do |(range, str)|
379
+ if str.empty? # is this a deletion?
380
+ "-#{range.to_range}"
381
+ elsif range.size == 0 # is this an insertion?
382
+ "+#{str.inspect}@#{range.begin_pos}"
383
+ else # it is a replacement
384
+ "^#{str.inspect}@#{range.to_range}"
385
+ end
386
+ end
387
+ parts << suffix if suffix
388
+ parts.join(', ')
389
+ end
390
+
298
391
  ACTIONS = %i[accept warn raise].freeze
299
392
  def check_policy_validity
300
393
  invalid = @policy.values - ACTIONS
@@ -310,7 +403,7 @@ module Parser
310
403
 
311
404
  def check_range_validity(range)
312
405
  if range.begin_pos < 0 || range.end_pos > @source_buffer.source.size
313
- raise IndexError, "The range #{range} is outside the bounds of the source"
406
+ raise IndexError, "The range #{range.to_range} is outside the bounds of the source"
314
407
  end
315
408
  range
316
409
  end
@@ -46,10 +46,49 @@ module Parser
46
46
  reps
47
47
  end
48
48
 
49
+ def nested_actions
50
+ actions = []
51
+ actions << [:wrap, @range, @insert_before, @insert_after] if !@insert_before.empty? ||
52
+ !@insert_after.empty?
53
+ actions << [:replace, @range, @replacement] if @replacement
54
+ actions.concat(@children.flat_map(&:nested_actions))
55
+ end
56
+
49
57
  def insertion?
50
58
  !insert_before.empty? || !insert_after.empty? || (replacement && !replacement.empty?)
51
59
  end
52
60
 
61
+ ##
62
+ # A root action has its range set to the whole source range, even
63
+ # though it typically do not act on that range.
64
+ # This method returns the action as if it was a child action with
65
+ # its range contracted.
66
+ # @return [Action]
67
+ def contract
68
+ raise 'Empty actions can not be contracted' if empty?
69
+ return self if insertion?
70
+ range = @range.with(
71
+ begin_pos: children.first.range.begin_pos,
72
+ end_pos: children.last.range.end_pos,
73
+ )
74
+ with(range: range)
75
+ end
76
+
77
+ ##
78
+ # @return [Action] that has been moved to the given source_buffer and with the given offset
79
+ # No check is done on validity of resulting range.
80
+ def moved(source_buffer, offset)
81
+ moved_range = ::Parser::Source::Range.new(
82
+ source_buffer,
83
+ @range.begin_pos + offset,
84
+ @range.end_pos + offset
85
+ )
86
+ with(
87
+ range: moved_range,
88
+ children: children.map { |child| child.moved(source_buffer, offset) }
89
+ )
90
+ end
91
+
53
92
  protected
54
93
 
55
94
  attr_reader :children
@@ -51,6 +51,10 @@ module Parser
51
51
  def declared_forward_args?
52
52
  declared?(FORWARD_ARGS)
53
53
  end
54
+
55
+ def empty?
56
+ @stack.empty?
57
+ end
54
58
  end
55
59
 
56
60
  end
@@ -8,6 +8,10 @@ module Parser
8
8
  push
9
9
  end
10
10
 
11
+ def empty?
12
+ @stack.empty?
13
+ end
14
+
11
15
  def push
12
16
  @stack << Set.new
13
17
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Parser
4
- VERSION = '2.7.1.3'
4
+ VERSION = '3.0.1.0'
5
5
  end
data/parser.gemspec CHANGED
@@ -20,29 +20,13 @@ Gem::Specification.new do |spec|
20
20
  'source_code_uri' => "https://github.com/whitequark/parser/tree/v#{spec.version}"
21
21
  }
22
22
 
23
- spec.files = `git ls-files`.split + %w(
24
- lib/parser/lexer.rb
25
- lib/parser/ruby18.rb
26
- lib/parser/ruby19.rb
27
- lib/parser/ruby20.rb
28
- lib/parser/ruby21.rb
29
- lib/parser/ruby22.rb
30
- lib/parser/ruby23.rb
31
- lib/parser/ruby24.rb
32
- lib/parser/ruby25.rb
33
- lib/parser/ruby26.rb
34
- lib/parser/ruby27.rb
35
- lib/parser/ruby28.rb
36
- lib/parser/macruby.rb
37
- lib/parser/rubymotion.rb
38
- )
23
+ spec.files = Dir['bin/*', 'lib/**/*.rb', 'parser.gemspec', 'LICENSE.txt']
39
24
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
40
- spec.test_files = spec.files.grep(%r{^test/})
41
25
  spec.require_paths = ['lib']
42
26
 
43
27
  spec.required_ruby_version = '>= 2.0.0'
44
28
 
45
- spec.add_dependency 'ast', '~> 2.4.0'
29
+ spec.add_dependency 'ast', '~> 2.4.1'
46
30
 
47
31
  spec.add_development_dependency 'bundler', '>= 1.15', '< 3.0.0'
48
32
  spec.add_development_dependency 'rake', '~> 13.0.1'