parser 2.7.1.4 → 3.0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -1
  3. data/lib/parser/all.rb +2 -1
  4. data/lib/parser/ast/processor.rb +3 -7
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +178 -29
  7. data/lib/parser/context.rb +4 -0
  8. data/lib/parser/current.rb +20 -11
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/lexer.rb +23780 -0
  11. data/lib/parser/macruby.rb +6149 -0
  12. data/lib/parser/max_numparam_stack.rb +13 -5
  13. data/lib/parser/messages.rb +3 -0
  14. data/lib/parser/meta.rb +7 -6
  15. data/lib/parser/ruby18.rb +5667 -0
  16. data/lib/parser/ruby19.rb +6092 -0
  17. data/lib/parser/ruby20.rb +6527 -0
  18. data/lib/parser/ruby21.rb +6578 -0
  19. data/lib/parser/ruby22.rb +6613 -0
  20. data/lib/parser/ruby23.rb +6624 -0
  21. data/lib/parser/ruby24.rb +6694 -0
  22. data/lib/parser/ruby25.rb +6662 -0
  23. data/lib/parser/ruby26.rb +6676 -0
  24. data/lib/parser/ruby27.rb +7862 -0
  25. data/lib/parser/ruby28.rb +8047 -0
  26. data/lib/parser/ruby30.rb +8060 -0
  27. data/lib/parser/ruby31.rb +8075 -0
  28. data/lib/parser/rubymotion.rb +6086 -0
  29. data/lib/parser/runner.rb +9 -4
  30. data/lib/parser/source/buffer.rb +50 -27
  31. data/lib/parser/source/comment.rb +14 -1
  32. data/lib/parser/source/comment/associator.rb +18 -5
  33. data/lib/parser/source/map/{endless_definition.rb → method_definition.rb} +5 -3
  34. data/lib/parser/source/range.rb +2 -2
  35. data/lib/parser/source/tree_rewriter.rb +27 -0
  36. data/lib/parser/static_environment.rb +4 -0
  37. data/lib/parser/variables_stack.rb +4 -0
  38. data/lib/parser/version.rb +1 -1
  39. data/parser.gemspec +1 -17
  40. metadata +9 -97
  41. data/.gitignore +0 -34
  42. data/.travis.yml +0 -40
  43. data/.yardopts +0 -21
  44. data/CHANGELOG.md +0 -1116
  45. data/CONTRIBUTING.md +0 -17
  46. data/Gemfile +0 -10
  47. data/README.md +0 -309
  48. data/Rakefile +0 -167
  49. data/ci/run_rubocop_specs +0 -14
  50. data/doc/AST_FORMAT.md +0 -2283
  51. data/doc/CUSTOMIZATION.md +0 -37
  52. data/doc/INTERNALS.md +0 -21
  53. data/doc/css/.gitkeep +0 -0
  54. data/doc/css/common.css +0 -68
  55. data/lib/parser/lexer.rl +0 -2550
  56. data/lib/parser/macruby.y +0 -2208
  57. data/lib/parser/ruby18.y +0 -1936
  58. data/lib/parser/ruby19.y +0 -2185
  59. data/lib/parser/ruby20.y +0 -2363
  60. data/lib/parser/ruby21.y +0 -2364
  61. data/lib/parser/ruby22.y +0 -2371
  62. data/lib/parser/ruby23.y +0 -2377
  63. data/lib/parser/ruby24.y +0 -2415
  64. data/lib/parser/ruby25.y +0 -2412
  65. data/lib/parser/ruby26.y +0 -2420
  66. data/lib/parser/ruby27.y +0 -2950
  67. data/lib/parser/ruby28.y +0 -3043
  68. data/lib/parser/rubymotion.y +0 -2192
  69. data/test/bug_163/fixtures/input.rb +0 -5
  70. data/test/bug_163/fixtures/output.rb +0 -5
  71. data/test/bug_163/rewriter.rb +0 -20
  72. data/test/helper.rb +0 -103
  73. data/test/parse_helper.rb +0 -323
  74. data/test/racc_coverage_helper.rb +0 -133
  75. data/test/test_ast_processor.rb +0 -32
  76. data/test/test_base.rb +0 -31
  77. data/test/test_current.rb +0 -31
  78. data/test/test_diagnostic.rb +0 -95
  79. data/test/test_diagnostic_engine.rb +0 -59
  80. data/test/test_encoding.rb +0 -99
  81. data/test/test_lexer.rb +0 -3617
  82. data/test/test_lexer_stack_state.rb +0 -78
  83. data/test/test_meta.rb +0 -12
  84. data/test/test_parse_helper.rb +0 -80
  85. data/test/test_parser.rb +0 -9780
  86. data/test/test_runner_parse.rb +0 -56
  87. data/test/test_runner_rewrite.rb +0 -47
  88. data/test/test_source_buffer.rb +0 -165
  89. data/test/test_source_comment.rb +0 -36
  90. data/test/test_source_comment_associator.rb +0 -399
  91. data/test/test_source_map.rb +0 -14
  92. data/test/test_source_range.rb +0 -192
  93. data/test/test_source_rewriter.rb +0 -541
  94. data/test/test_source_rewriter_action.rb +0 -46
  95. data/test/test_source_tree_rewriter.rb +0 -361
  96. data/test/test_static_environment.rb +0 -45
  97. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  98. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  99. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if(true)
4
- puts "Hello, world!"
5
- end
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if true
4
- puts "Hello, world!"
5
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class Rewriter < Parser::Rewriter
4
- def on_if(node)
5
- # Crude, totally-not-usable-in-the-real-world code to remove optional
6
- # parens from control keywords.
7
- #
8
- # In a perfect test scenario we'd simply make this a no-op, to demonstrate
9
- # that the bug happens when any rewriter is loaded regardless of whether it
10
- # actually changes anything but that makes assertions much harder to get
11
- # right. It's much easier to just show that the file did, or did not
12
- # get changed.
13
- if node.children[0].type == :begin
14
- replace node.children[0].loc.begin, ' '
15
- remove node.children[0].loc.end
16
- end
17
-
18
- super
19
- end
20
- end
data/test/helper.rb DELETED
@@ -1,103 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'tempfile'
4
- require 'simplecov'
5
-
6
- if ENV.include?('COVERAGE') && SimpleCov.usable?
7
- require_relative 'racc_coverage_helper'
8
-
9
- RaccCoverage.start(
10
- %w(
11
- ruby18.y
12
- ruby19.y
13
- ruby20.y
14
- ruby21.y
15
- ruby22.y
16
- ruby23.y
17
- ruby24.y
18
- ruby25.y
19
- ruby26.y
20
- ruby27.y
21
- ruby28.y
22
- ),
23
- File.expand_path('../../lib/parser', __FILE__))
24
-
25
- # Report results faster.
26
- at_exit { RaccCoverage.stop }
27
-
28
- SimpleCov.start do
29
- self.formatter = SimpleCov::Formatter::MultiFormatter.new(
30
- SimpleCov::Formatter::HTMLFormatter
31
- )
32
-
33
- add_group 'Grammars' do |source_file|
34
- source_file.filename =~ %r{\.y$}
35
- end
36
-
37
- # Exclude the testsuite itself.
38
- add_filter '/test/'
39
-
40
- # Exclude generated files.
41
- add_filter do |source_file|
42
- source_file.filename =~ %r{/lib/parser/(lexer|ruby\d+|macruby|rubymotion)\.rb$}
43
- end
44
- end
45
- end
46
-
47
- # minitest/autorun must go after SimpleCov to preserve
48
- # correct order of at_exit hooks.
49
- require 'minitest/autorun'
50
-
51
- $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
52
- require 'parser'
53
-
54
- module NodeCollector
55
- extend self
56
- attr_accessor :callbacks, :nodes
57
- self.callbacks = []
58
- self.nodes = []
59
-
60
- def check
61
- @callbacks.each do |callback|
62
- @nodes.each { |node| callback.call(node) }
63
- end
64
- puts "#{callbacks.size} additional tests on #{nodes.size} nodes ran successfully"
65
- end
66
-
67
- Minitest.after_run { check }
68
- end
69
-
70
- def for_each_node(&block)
71
- NodeCollector.callbacks << block
72
- end
73
-
74
- class Parser::AST::Node
75
- def initialize(type, *)
76
- NodeCollector.nodes << self
77
- super
78
- end
79
- end
80
-
81
- # Special test extension that records a context of the parser
82
- # for any node that is created
83
- module NodeContextExt
84
- module NodeExt
85
- attr_reader :context
86
-
87
- def assign_properties(properties)
88
- super
89
-
90
- if (context = properties[:context])
91
- @context = context
92
- end
93
- end
94
- end
95
- Parser::AST::Node.prepend(NodeExt)
96
-
97
- module BuilderExt
98
- def n(type, children, source_map)
99
- super.updated(nil, nil, context: @parser.context.stack.dup)
100
- end
101
- end
102
- Parser::Builders::Default.prepend(BuilderExt)
103
- end
data/test/parse_helper.rb DELETED
@@ -1,323 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module ParseHelper
4
- include AST::Sexp
5
-
6
- require 'parser/all'
7
- require 'parser/macruby'
8
- require 'parser/rubymotion'
9
-
10
- ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 mac ios)
11
-
12
- def setup
13
- @diagnostics = []
14
-
15
- super if defined?(super)
16
- end
17
-
18
- def parser_for_ruby_version(version)
19
- case version
20
- when '1.8' then parser = Parser::Ruby18.new
21
- when '1.9' then parser = Parser::Ruby19.new
22
- when '2.0' then parser = Parser::Ruby20.new
23
- when '2.1' then parser = Parser::Ruby21.new
24
- when '2.2' then parser = Parser::Ruby22.new
25
- when '2.3' then parser = Parser::Ruby23.new
26
- when '2.4' then parser = Parser::Ruby24.new
27
- when '2.5' then parser = Parser::Ruby25.new
28
- when '2.6' then parser = Parser::Ruby26.new
29
- when '2.7' then parser = Parser::Ruby27.new
30
- when '2.8' then parser = Parser::Ruby28.new
31
- when 'mac' then parser = Parser::MacRuby.new
32
- when 'ios' then parser = Parser::RubyMotion.new
33
- else raise "Unrecognized Ruby version #{version}"
34
- end
35
-
36
- parser.diagnostics.consumer = lambda do |diagnostic|
37
- @diagnostics << diagnostic
38
- end
39
-
40
- parser
41
- end
42
-
43
- def with_versions(versions)
44
- (versions & ALL_VERSIONS).each do |version|
45
- @diagnostics.clear
46
-
47
- parser = parser_for_ruby_version(version)
48
- yield version, parser
49
- end
50
- end
51
-
52
- def assert_source_range(begin_pos, end_pos, range, version, what)
53
- assert range.is_a?(Parser::Source::Range),
54
- "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
55
-
56
- assert_equal begin_pos, range.begin_pos,
57
- "(#{version}) begin of #{what}"
58
-
59
- assert_equal end_pos, range.end_pos,
60
- "(#{version}) end of #{what}"
61
- end
62
-
63
- # Use like this:
64
- # ~~~
65
- # assert_parses(
66
- # s(:send, s(:lit, 10), :+, s(:lit, 20))
67
- # %q{10 + 20},
68
- # %q{~~~~~~~ expression
69
- # | ^ operator
70
- # | ~~ expression (lit)
71
- # },
72
- # %w(1.8 1.9) # optional
73
- # )
74
- # ~~~
75
- def assert_parses(ast, code, source_maps='', versions=ALL_VERSIONS)
76
- with_versions(versions) do |version, parser|
77
- try_parsing(ast, code, parser, source_maps, version)
78
- end
79
-
80
- # Also try parsing with lexer set to use UTF-32LE internally
81
- with_versions(versions) do |version, parser|
82
- parser.instance_eval { @lexer.force_utf32 = true }
83
- try_parsing(ast, code, parser, source_maps, version)
84
- end
85
- end
86
-
87
- def try_parsing(ast, code, parser, source_maps, version)
88
- source_file = Parser::Source::Buffer.new('(assert_parses)', source: code)
89
-
90
- begin
91
- parsed_ast = parser.parse(source_file)
92
- rescue => exc
93
- backtrace = exc.backtrace
94
- Exception.instance_method(:initialize).bind(exc).
95
- call("(#{version}) #{exc.message}")
96
- exc.set_backtrace(backtrace)
97
- raise
98
- end
99
-
100
- if ast.nil?
101
- assert_nil parsed_ast, "(#{version}) AST equality"
102
- return
103
- end
104
-
105
- assert_equal ast, parsed_ast,
106
- "(#{version}) AST equality"
107
-
108
- parse_source_map_descriptions(source_maps) do |begin_pos, end_pos, map_field, ast_path, line|
109
-
110
- astlet = traverse_ast(parsed_ast, ast_path)
111
-
112
- if astlet.nil?
113
- # This is a testsuite bug.
114
- raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
115
- end
116
-
117
- assert astlet.frozen?
118
-
119
- assert astlet.location.respond_to?(map_field),
120
- "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
121
-
122
- range = astlet.location.send(map_field)
123
-
124
- assert_source_range(begin_pos, end_pos, range, version, line.inspect)
125
- end
126
-
127
- assert parser.instance_eval { @lexer }.cmdarg.empty?,
128
- "(#{version}) expected cmdarg to be empty after parsing"
129
-
130
- assert_equal 0, parser.instance_eval { @lexer.instance_eval { @paren_nest } },
131
- "(#{version}) expected paren_nest to be 0 after parsing"
132
- end
133
-
134
- # Use like this:
135
- # ~~~
136
- # assert_diagnoses(
137
- # [:warning, :ambiguous_prefix, { prefix: '*' }],
138
- # %q{foo *bar},
139
- # %q{ ^ location
140
- # | ~~~ highlights (0)})
141
- # ~~~
142
- def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
143
- with_versions(versions) do |version, parser|
144
- source_file = Parser::Source::Buffer.new('(assert_diagnoses)', source: code)
145
-
146
- begin
147
- parser = parser.parse(source_file)
148
- rescue Parser::SyntaxError
149
- # do nothing; the diagnostic was reported
150
- end
151
-
152
- assert_equal 1, @diagnostics.count,
153
- "(#{version}) emits a single diagnostic, not\n" \
154
- "#{@diagnostics.map(&:render).join("\n")}"
155
-
156
- emitted_diagnostic = @diagnostics.first
157
-
158
- level, reason, arguments = diagnostic
159
- arguments ||= {}
160
- message = Parser::Messages.compile(reason, arguments)
161
-
162
- assert_equal level, emitted_diagnostic.level
163
- assert_equal reason, emitted_diagnostic.reason
164
- assert_equal arguments, emitted_diagnostic.arguments
165
- assert_equal message, emitted_diagnostic.message
166
-
167
- parse_source_map_descriptions(source_maps) do |begin_pos, end_pos, map_field, ast_path, line|
168
-
169
- case map_field
170
- when 'location'
171
- assert_source_range begin_pos, end_pos,
172
- emitted_diagnostic.location,
173
- version, 'location'
174
-
175
- when 'highlights'
176
- index = ast_path.first.to_i
177
-
178
- assert_source_range begin_pos, end_pos,
179
- emitted_diagnostic.highlights[index],
180
- version, "#{index}th highlight"
181
-
182
- else
183
- raise "Unknown diagnostic range #{map_field}"
184
- end
185
- end
186
- end
187
- end
188
-
189
- # Use like this:
190
- # ~~~
191
- # assert_diagnoses_many(
192
- # [
193
- # [:warning, :ambiguous_literal],
194
- # [:error, :unexpected_token, { :token => :tLCURLY }]
195
- # ],
196
- # %q{m /foo/ {}},
197
- # SINCE_2_4)
198
- # ~~~
199
- def assert_diagnoses_many(diagnostics, code, versions=ALL_VERSIONS)
200
- with_versions(versions) do |version, parser|
201
- source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)', source: code)
202
-
203
- begin
204
- parser = parser.parse(source_file)
205
- rescue Parser::SyntaxError
206
- # do nothing; the diagnostic was reported
207
- end
208
-
209
- assert_equal diagnostics.count, @diagnostics.count
210
-
211
- diagnostics.zip(@diagnostics) do |expected_diagnostic, actual_diagnostic|
212
- level, reason, arguments = expected_diagnostic
213
- arguments ||= {}
214
- message = Parser::Messages.compile(reason, arguments)
215
-
216
- assert_equal level, actual_diagnostic.level
217
- assert_equal reason, actual_diagnostic.reason
218
- assert_equal arguments, actual_diagnostic.arguments
219
- assert_equal message, actual_diagnostic.message
220
- end
221
- end
222
- end
223
-
224
- def refute_diagnoses(code, versions=ALL_VERSIONS)
225
- with_versions(versions) do |version, parser|
226
- source_file = Parser::Source::Buffer.new('(refute_diagnoses)', source: code)
227
-
228
- begin
229
- parser = parser.parse(source_file)
230
- rescue Parser::SyntaxError
231
- # do nothing; the diagnostic was reported
232
- end
233
-
234
- assert_empty @diagnostics,
235
- "(#{version}) emits no diagnostics, not\n" \
236
- "#{@diagnostics.map(&:render).join("\n")}"
237
- end
238
- end
239
-
240
- def assert_context(context, code, versions=ALL_VERSIONS)
241
- with_versions(versions) do |version, parser|
242
- source_file = Parser::Source::Buffer.new('(assert_context)', source: code)
243
-
244
- parsed_ast = parser.parse(source_file)
245
-
246
- nodes = find_matching_nodes(parsed_ast) { |node| node.type == :send && node.children[1] == :get_context }
247
- assert_equal 1, nodes.count, "there must exactly 1 `get_context()` call"
248
-
249
- node = nodes.first
250
- assert_equal context, node.context, "(#{version}) expect parsing context to match"
251
- end
252
- end
253
-
254
- SOURCE_MAP_DESCRIPTION_RE =
255
- /(?x)
256
- ^(?# $1 skip) ^(\s*)
257
- (?# $2 highlight) ([~\^]+)
258
- \s+
259
- (?# $3 source_map_field) ([a-z_]+)
260
- (?# $5 ast_path) (\s+\(([a-z_.\/0-9]+)\))?
261
- $/
262
-
263
- def parse_source_map_descriptions(descriptions)
264
- unless block_given?
265
- return to_enum(:parse_source_map_descriptions, descriptions)
266
- end
267
-
268
- descriptions.each_line do |line|
269
- # Remove leading " |", if it exists.
270
- line = line.sub(/^\s*\|/, '').rstrip
271
-
272
- next if line.empty?
273
-
274
- if (match = SOURCE_MAP_DESCRIPTION_RE.match(line))
275
- begin_pos = match[1].length
276
- end_pos = begin_pos + match[2].length
277
- source_map_field = match[3]
278
-
279
- if match[5]
280
- ast_path = match[5].split('.')
281
- else
282
- ast_path = []
283
- end
284
-
285
- yield begin_pos, end_pos, source_map_field, ast_path, line
286
- else
287
- raise "Cannot parse source map description line: #{line.inspect}."
288
- end
289
- end
290
- end
291
-
292
- def traverse_ast(ast, path)
293
- path.inject(ast) do |astlet, path_component|
294
- # Split "dstr/2" to :dstr and 1
295
- type_str, index_str = path_component.split('/')
296
-
297
- type = type_str.to_sym
298
-
299
- if index_str.nil?
300
- index = 0
301
- else
302
- index = index_str.to_i - 1
303
- end
304
-
305
- matching_children = \
306
- astlet.children.select do |child|
307
- AST::Node === child && child.type == type
308
- end
309
-
310
- matching_children[index]
311
- end
312
- end
313
-
314
- def find_matching_nodes(ast, &block)
315
- return [] unless ast.is_a?(AST::Node)
316
-
317
- result = []
318
- result << ast if block.call(ast)
319
- ast.children.each { |child| result += find_matching_nodes(child, &block) }
320
-
321
- result
322
- end
323
- end