parser 2.7.1.3 → 3.0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -1
  3. data/lib/parser/all.rb +1 -1
  4. data/lib/parser/ast/processor.rb +5 -7
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +225 -29
  7. data/lib/parser/context.rb +5 -0
  8. data/lib/parser/current.rb +11 -11
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/lexer.rb +23780 -0
  11. data/lib/parser/macruby.rb +6149 -0
  12. data/lib/parser/max_numparam_stack.rb +13 -5
  13. data/lib/parser/messages.rb +3 -0
  14. data/lib/parser/meta.rb +8 -7
  15. data/lib/parser/ruby18.rb +5667 -0
  16. data/lib/parser/ruby19.rb +6092 -0
  17. data/lib/parser/ruby20.rb +6527 -0
  18. data/lib/parser/ruby21.rb +6578 -0
  19. data/lib/parser/ruby22.rb +6613 -0
  20. data/lib/parser/ruby23.rb +6624 -0
  21. data/lib/parser/ruby24.rb +6694 -0
  22. data/lib/parser/ruby25.rb +6662 -0
  23. data/lib/parser/ruby26.rb +6676 -0
  24. data/lib/parser/ruby27.rb +7862 -0
  25. data/lib/parser/ruby28.rb +8047 -0
  26. data/lib/parser/ruby30.rb +8060 -0
  27. data/lib/parser/rubymotion.rb +6086 -0
  28. data/lib/parser/runner.rb +4 -4
  29. data/lib/parser/source/buffer.rb +50 -27
  30. data/lib/parser/source/comment.rb +1 -1
  31. data/lib/parser/source/comment/associator.rb +1 -1
  32. data/lib/parser/source/map/{endless_definition.rb → method_definition.rb} +5 -3
  33. data/lib/parser/source/range.rb +3 -3
  34. data/lib/parser/source/tree_rewriter.rb +94 -1
  35. data/lib/parser/source/tree_rewriter/action.rb +39 -0
  36. data/lib/parser/static_environment.rb +4 -0
  37. data/lib/parser/variables_stack.rb +4 -0
  38. data/lib/parser/version.rb +1 -1
  39. data/parser.gemspec +2 -18
  40. metadata +13 -102
  41. data/.gitignore +0 -34
  42. data/.travis.yml +0 -40
  43. data/.yardopts +0 -21
  44. data/CHANGELOG.md +0 -1101
  45. data/CONTRIBUTING.md +0 -17
  46. data/Gemfile +0 -10
  47. data/README.md +0 -308
  48. data/Rakefile +0 -167
  49. data/ci/run_rubocop_specs +0 -14
  50. data/doc/AST_FORMAT.md +0 -2229
  51. data/doc/CUSTOMIZATION.md +0 -37
  52. data/doc/INTERNALS.md +0 -21
  53. data/doc/css/.gitkeep +0 -0
  54. data/doc/css/common.css +0 -68
  55. data/lib/parser/lexer.rl +0 -2543
  56. data/lib/parser/macruby.y +0 -2198
  57. data/lib/parser/ruby18.y +0 -1934
  58. data/lib/parser/ruby19.y +0 -2175
  59. data/lib/parser/ruby20.y +0 -2353
  60. data/lib/parser/ruby21.y +0 -2357
  61. data/lib/parser/ruby22.y +0 -2364
  62. data/lib/parser/ruby23.y +0 -2370
  63. data/lib/parser/ruby24.y +0 -2408
  64. data/lib/parser/ruby25.y +0 -2405
  65. data/lib/parser/ruby26.y +0 -2413
  66. data/lib/parser/ruby27.y +0 -2941
  67. data/lib/parser/ruby28.y +0 -3016
  68. data/lib/parser/rubymotion.y +0 -2182
  69. data/test/bug_163/fixtures/input.rb +0 -5
  70. data/test/bug_163/fixtures/output.rb +0 -5
  71. data/test/bug_163/rewriter.rb +0 -20
  72. data/test/helper.rb +0 -79
  73. data/test/parse_helper.rb +0 -313
  74. data/test/racc_coverage_helper.rb +0 -133
  75. data/test/test_ast_processor.rb +0 -32
  76. data/test/test_base.rb +0 -31
  77. data/test/test_current.rb +0 -31
  78. data/test/test_diagnostic.rb +0 -95
  79. data/test/test_diagnostic_engine.rb +0 -59
  80. data/test/test_encoding.rb +0 -99
  81. data/test/test_lexer.rb +0 -3617
  82. data/test/test_lexer_stack_state.rb +0 -78
  83. data/test/test_meta.rb +0 -12
  84. data/test/test_parse_helper.rb +0 -80
  85. data/test/test_parser.rb +0 -9596
  86. data/test/test_runner_parse.rb +0 -56
  87. data/test/test_runner_rewrite.rb +0 -47
  88. data/test/test_source_buffer.rb +0 -165
  89. data/test/test_source_comment.rb +0 -36
  90. data/test/test_source_comment_associator.rb +0 -399
  91. data/test/test_source_map.rb +0 -14
  92. data/test/test_source_range.rb +0 -192
  93. data/test/test_source_rewriter.rb +0 -541
  94. data/test/test_source_rewriter_action.rb +0 -46
  95. data/test/test_source_tree_rewriter.rb +0 -263
  96. data/test/test_static_environment.rb +0 -45
  97. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  98. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  99. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
data/lib/parser/runner.rb CHANGED
@@ -37,7 +37,7 @@ module Parser
37
37
 
38
38
  private
39
39
 
40
- LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0].freeze
40
+ LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0 forward_arg kwargs match_pattern].freeze
41
41
 
42
42
  def runner_name
43
43
  raise NotImplementedError, "implement #{self.class}##{__callee__}"
@@ -113,9 +113,9 @@ module Parser
113
113
  @parser_class = Parser::Ruby27
114
114
  end
115
115
 
116
- opts.on '--28', 'Parse as Ruby 2.8 would' do
117
- require 'parser/ruby28'
118
- @parser_class = Parser::Ruby28
116
+ opts.on '--30', 'Parse as Ruby 3.0 would' do
117
+ require 'parser/ruby30'
118
+ @parser_class = Parser::Ruby30
119
119
  end
120
120
 
121
121
  opts.on '--mac', 'Parse as MacRuby 0.12 would' do
@@ -114,8 +114,7 @@ module Parser
114
114
  @slice_source = nil
115
115
 
116
116
  # Cache for fast lookup
117
- @line_for_position = {}
118
- @column_for_position = {}
117
+ @line_index_for_position = {}
119
118
 
120
119
  self.source = source if source
121
120
  end
@@ -207,9 +206,10 @@ module Parser
207
206
  # @return [[Integer, Integer]] `[line, column]`
208
207
  #
209
208
  def decompose_position(position)
210
- line_no, line_begin = line_for(position)
209
+ line_index = line_index_for_position(position)
210
+ line_begin = line_begins[line_index]
211
211
 
212
- [ @first_line + line_no, position - line_begin ]
212
+ [ @first_line + line_index , position - line_begin ]
213
213
  end
214
214
 
215
215
  ##
@@ -220,10 +220,7 @@ module Parser
220
220
  # @api private
221
221
  #
222
222
  def line_for_position(position)
223
- @line_for_position[position] ||= begin
224
- line_no, _ = line_for(position)
225
- @first_line + line_no
226
- end
223
+ line_index_for_position(position) + @first_line
227
224
  end
228
225
 
229
226
  ##
@@ -234,10 +231,8 @@ module Parser
234
231
  # @api private
235
232
  #
236
233
  def column_for_position(position)
237
- @column_for_position[position] ||= begin
238
- _, line_begin = line_for(position)
239
- position - line_begin
240
- end
234
+ line_index = line_index_for_position(position)
235
+ position - line_begins[line_index]
241
236
  end
242
237
 
243
238
  ##
@@ -278,15 +273,13 @@ module Parser
278
273
  # @raise [IndexError] if `lineno` is out of bounds
279
274
  #
280
275
  def line_range(lineno)
281
- index = lineno - @first_line + 1
282
- if index <= 0 || index > line_begins.size
276
+ index = lineno - @first_line
277
+ if index < 0 || index + 1 >= line_begins.size
283
278
  raise IndexError, 'Parser::Source::Buffer: range for line ' \
284
279
  "#{lineno} requested, valid line numbers are #{@first_line}.." \
285
- "#{@first_line + line_begins.size - 1}"
286
- elsif index == line_begins.size
287
- Range.new(self, line_begins[-index][1], @source.size)
280
+ "#{@first_line + line_begins.size - 2}"
288
281
  else
289
- Range.new(self, line_begins[-index][1], line_begins[-index - 1][1] - 1)
282
+ Range.new(self, line_begins[index], line_begins[index + 1] - 1)
290
283
  end
291
284
  end
292
285
 
@@ -303,27 +296,57 @@ module Parser
303
296
  # @return [Integer]
304
297
  #
305
298
  def last_line
306
- line_begins.size + @first_line - 1
299
+ line_begins.size + @first_line - 2
300
+ end
301
+
302
+ # :nodoc:
303
+ def freeze
304
+ source_lines; line_begins; source_range # build cache
305
+ super
306
+ end
307
+
308
+ # :nodoc:
309
+ def inspect
310
+ "#<#{self.class} #{name}>"
307
311
  end
308
312
 
309
313
  private
310
314
 
315
+ # @returns [0, line_begin_of_line_1, ..., source.size + 1]
311
316
  def line_begins
312
- unless @line_begins
313
- @line_begins, index = [ [ 0, 0 ] ], 0
314
-
317
+ @line_begins ||= begin
318
+ begins = [0]
319
+ index = 0
315
320
  while index = @source.index("\n".freeze, index)
316
321
  index += 1
317
- @line_begins.unshift [ @line_begins.length, index ]
322
+ begins << index
318
323
  end
324
+ begins << @source.size + 1
325
+ begins
319
326
  end
327
+ end
320
328
 
321
- @line_begins
329
+ # @returns 0-based line index of position
330
+ def line_index_for_position(position)
331
+ @line_index_for_position[position] || begin
332
+ index = bsearch(line_begins, position) - 1
333
+ @line_index_for_position[position] = index unless @line_index_for_position.frozen?
334
+ index
335
+ end
322
336
  end
323
337
 
324
- def line_for(position)
325
- line_begins.bsearch do |line, line_begin|
326
- line_begin <= position
338
+ if Array.method_defined?(:bsearch_index) # RUBY_VERSION >= 2.3
339
+ def bsearch(line_begins, position)
340
+ line_begins.bsearch_index do |line_begin|
341
+ position < line_begin
342
+ end || line_begins.size - 1 # || only for out of bound values
343
+ end
344
+ else
345
+ def bsearch(line_begins, position)
346
+ @line_range ||= 0...line_begins.size
347
+ @line_range.bsearch do |i|
348
+ position < line_begins[i]
349
+ end || line_begins.size - 1 # || only for out of bound values
327
350
  end
328
351
  end
329
352
  end
@@ -10,7 +10,7 @@ module Parser
10
10
  # @return [String]
11
11
  #
12
12
  # @!attribute [r] location
13
- # @return [Parser::Source::Map]
13
+ # @return [Parser::Source::Range]
14
14
  #
15
15
  # @api public
16
16
  #
@@ -107,7 +107,7 @@ module Parser
107
107
 
108
108
  private
109
109
 
110
- POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post].freeze
110
+ POSTFIX_TYPES = Set[:if, :while, :while_post, :until, :until_post, :masgn].freeze
111
111
  def children_in_source_order(node)
112
112
  if POSTFIX_TYPES.include?(node.type)
113
113
  # All these types have either nodes with expressions, or `nil`
@@ -3,19 +3,21 @@
3
3
  module Parser
4
4
  module Source
5
5
 
6
- class Map::EndlessDefinition < Map
6
+ class Map::MethodDefinition < Map
7
7
  attr_reader :keyword
8
8
  attr_reader :operator
9
9
  attr_reader :name
10
+ attr_reader :end
10
11
  attr_reader :assignment
11
12
 
12
- def initialize(keyword_l, operator_l, name_l, assignment_l, body_l)
13
+ def initialize(keyword_l, operator_l, name_l, end_l, assignment_l, body_l)
13
14
  @keyword = keyword_l
14
15
  @operator = operator_l
15
16
  @name = name_l
17
+ @end = end_l
16
18
  @assignment = assignment_l
17
19
 
18
- super(@keyword.join(body_l))
20
+ super(@keyword.join(end_l || body_l))
19
21
  end
20
22
  end
21
23
 
@@ -13,7 +13,7 @@ module Parser
13
13
  # ^^
14
14
  #
15
15
  # @!attribute [r] source_buffer
16
- # @return [Parser::Diagnostic::Engine]
16
+ # @return [Parser::Source::Buffer]
17
17
  #
18
18
  # @!attribute [r] begin_pos
19
19
  # @return [Integer] index of the first character in the range
@@ -112,11 +112,11 @@ module Parser
112
112
  # @raise RangeError
113
113
  #
114
114
  def column_range
115
- if self.begin.line != self.end.line
115
+ if line != last_line
116
116
  raise RangeError, "#{self.inspect} spans more than one line"
117
117
  end
118
118
 
119
- self.begin.column...self.end.column
119
+ column...last_column
120
120
  end
121
121
 
122
122
  ##
@@ -129,6 +129,8 @@ module Parser
129
129
  ##
130
130
  # Merges the updates of argument with the receiver.
131
131
  # Policies of the receiver are used.
132
+ # This action is atomic in that it won't change the receiver
133
+ # unless it succeeds.
132
134
  #
133
135
  # @param [Rewriter] with
134
136
  # @return [Rewriter] self
@@ -154,6 +156,32 @@ module Parser
154
156
  dup.merge!(with)
155
157
  end
156
158
 
159
+ ##
160
+ # For special cases where one needs to merge a rewriter attached to a different source_buffer
161
+ # or that needs to be offset. Policies of the receiver are used.
162
+ #
163
+ # @param [TreeRewriter] rewriter from different source_buffer
164
+ # @param [Integer] offset
165
+ # @return [Rewriter] self
166
+ # @raise [IndexError] if action ranges (once offset) don't fit the current buffer
167
+ #
168
+ def import!(foreign_rewriter, offset: 0)
169
+ return self if foreign_rewriter.empty?
170
+
171
+ contracted = foreign_rewriter.action_root.contract
172
+ merge_effective_range = ::Parser::Source::Range.new(
173
+ @source_buffer,
174
+ contracted.range.begin_pos + offset,
175
+ contracted.range.end_pos + offset,
176
+ )
177
+ check_range_validity(merge_effective_range)
178
+
179
+ merge_with = contracted.moved(@source_buffer, offset)
180
+
181
+ @action_root = @action_root.combine(merge_with)
182
+ self
183
+ end
184
+
157
185
  ##
158
186
  # Replaces the code of the source range `range` with `content`.
159
187
  #
@@ -234,6 +262,44 @@ module Parser
234
262
  chunks.join
235
263
  end
236
264
 
265
+ ##
266
+ # Returns a representation of the rewriter as an ordered list of replacements.
267
+ #
268
+ # rewriter.as_replacements # => [ [1...1, '('],
269
+ # [2...4, 'foo'],
270
+ # [5...6, ''],
271
+ # [6...6, '!'],
272
+ # [10...10, ')'],
273
+ # ]
274
+ #
275
+ # This representation is sufficient to recreate the result of `process` but it is
276
+ # not sufficient to recreate completely the rewriter for further merging/actions.
277
+ # See `as_nested_actions`
278
+ #
279
+ # @return [Array<Range, String>] an ordered list of pairs of range & replacement
280
+ #
281
+ def as_replacements
282
+ @action_root.ordered_replacements
283
+ end
284
+
285
+ ##
286
+ # Returns a representation of the rewriter as nested insertions (:wrap) and replacements.
287
+ #
288
+ # rewriter.as_actions # =>[ [:wrap, 1...10, '(', ')'],
289
+ # [:wrap, 2...6, '', '!'], # aka "insert_after"
290
+ # [:replace, 2...4, 'foo'],
291
+ # [:replace, 5...6, ''], # aka "removal"
292
+ # ],
293
+ #
294
+ # Contrary to `as_replacements`, this representation is sufficient to recreate exactly
295
+ # the rewriter.
296
+ #
297
+ # @return [Array<(Symbol, Range, String{, String})>]
298
+ #
299
+ def as_nested_actions
300
+ @action_root.nested_actions
301
+ end
302
+
237
303
  ##
238
304
  # Provides a protected block where a sequence of multiple rewrite actions
239
305
  # are handled atomically. If any of the actions failed by clobbering,
@@ -264,6 +330,11 @@ module Parser
264
330
  @in_transaction
265
331
  end
266
332
 
333
+ # :nodoc:
334
+ def inspect
335
+ "#<#{self.class} #{source_buffer.name}: #{action_summary}>"
336
+ end
337
+
267
338
  ##
268
339
  # @api private
269
340
  # @deprecated Use insert_after or wrap
@@ -295,6 +366,28 @@ module Parser
295
366
 
296
367
  private
297
368
 
369
+ def action_summary
370
+ replacements = as_replacements
371
+ case replacements.size
372
+ when 0 then return 'empty'
373
+ when 1..3 then #ok
374
+ else
375
+ replacements = replacements.first(3)
376
+ suffix = '…'
377
+ end
378
+ parts = replacements.map do |(range, str)|
379
+ if str.empty? # is this a deletion?
380
+ "-#{range.to_range}"
381
+ elsif range.size == 0 # is this an insertion?
382
+ "+#{str.inspect}@#{range.begin_pos}"
383
+ else # it is a replacement
384
+ "^#{str.inspect}@#{range.to_range}"
385
+ end
386
+ end
387
+ parts << suffix if suffix
388
+ parts.join(', ')
389
+ end
390
+
298
391
  ACTIONS = %i[accept warn raise].freeze
299
392
  def check_policy_validity
300
393
  invalid = @policy.values - ACTIONS
@@ -310,7 +403,7 @@ module Parser
310
403
 
311
404
  def check_range_validity(range)
312
405
  if range.begin_pos < 0 || range.end_pos > @source_buffer.source.size
313
- raise IndexError, "The range #{range} is outside the bounds of the source"
406
+ raise IndexError, "The range #{range.to_range} is outside the bounds of the source"
314
407
  end
315
408
  range
316
409
  end
@@ -46,10 +46,49 @@ module Parser
46
46
  reps
47
47
  end
48
48
 
49
+ def nested_actions
50
+ actions = []
51
+ actions << [:wrap, @range, @insert_before, @insert_after] if !@insert_before.empty? ||
52
+ !@insert_after.empty?
53
+ actions << [:replace, @range, @replacement] if @replacement
54
+ actions.concat(@children.flat_map(&:nested_actions))
55
+ end
56
+
49
57
  def insertion?
50
58
  !insert_before.empty? || !insert_after.empty? || (replacement && !replacement.empty?)
51
59
  end
52
60
 
61
+ ##
62
+ # A root action has its range set to the whole source range, even
63
+ # though it typically do not act on that range.
64
+ # This method returns the action as if it was a child action with
65
+ # its range contracted.
66
+ # @return [Action]
67
+ def contract
68
+ raise 'Empty actions can not be contracted' if empty?
69
+ return self if insertion?
70
+ range = @range.with(
71
+ begin_pos: children.first.range.begin_pos,
72
+ end_pos: children.last.range.end_pos,
73
+ )
74
+ with(range: range)
75
+ end
76
+
77
+ ##
78
+ # @return [Action] that has been moved to the given source_buffer and with the given offset
79
+ # No check is done on validity of resulting range.
80
+ def moved(source_buffer, offset)
81
+ moved_range = ::Parser::Source::Range.new(
82
+ source_buffer,
83
+ @range.begin_pos + offset,
84
+ @range.end_pos + offset
85
+ )
86
+ with(
87
+ range: moved_range,
88
+ children: children.map { |child| child.moved(source_buffer, offset) }
89
+ )
90
+ end
91
+
53
92
  protected
54
93
 
55
94
  attr_reader :children
@@ -51,6 +51,10 @@ module Parser
51
51
  def declared_forward_args?
52
52
  declared?(FORWARD_ARGS)
53
53
  end
54
+
55
+ def empty?
56
+ @stack.empty?
57
+ end
54
58
  end
55
59
 
56
60
  end
@@ -8,6 +8,10 @@ module Parser
8
8
  push
9
9
  end
10
10
 
11
+ def empty?
12
+ @stack.empty?
13
+ end
14
+
11
15
  def push
12
16
  @stack << Set.new
13
17
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Parser
4
- VERSION = '2.7.1.3'
4
+ VERSION = '3.0.1.0'
5
5
  end
data/parser.gemspec CHANGED
@@ -20,29 +20,13 @@ Gem::Specification.new do |spec|
20
20
  'source_code_uri' => "https://github.com/whitequark/parser/tree/v#{spec.version}"
21
21
  }
22
22
 
23
- spec.files = `git ls-files`.split + %w(
24
- lib/parser/lexer.rb
25
- lib/parser/ruby18.rb
26
- lib/parser/ruby19.rb
27
- lib/parser/ruby20.rb
28
- lib/parser/ruby21.rb
29
- lib/parser/ruby22.rb
30
- lib/parser/ruby23.rb
31
- lib/parser/ruby24.rb
32
- lib/parser/ruby25.rb
33
- lib/parser/ruby26.rb
34
- lib/parser/ruby27.rb
35
- lib/parser/ruby28.rb
36
- lib/parser/macruby.rb
37
- lib/parser/rubymotion.rb
38
- )
23
+ spec.files = Dir['bin/*', 'lib/**/*.rb', 'parser.gemspec', 'LICENSE.txt']
39
24
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
40
- spec.test_files = spec.files.grep(%r{^test/})
41
25
  spec.require_paths = ['lib']
42
26
 
43
27
  spec.required_ruby_version = '>= 2.0.0'
44
28
 
45
- spec.add_dependency 'ast', '~> 2.4.0'
29
+ spec.add_dependency 'ast', '~> 2.4.1'
46
30
 
47
31
  spec.add_development_dependency 'bundler', '>= 1.15', '< 3.0.0'
48
32
  spec.add_development_dependency 'rake', '~> 13.0.1'