parser 2.7.1.1 → 3.0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +1 -0
  3. data/lib/parser/all.rb +2 -0
  4. data/lib/parser/ast/processor.rb +5 -0
  5. data/lib/parser/base.rb +7 -5
  6. data/lib/parser/builders/default.rb +263 -23
  7. data/lib/parser/context.rb +5 -0
  8. data/lib/parser/current.rb +24 -6
  9. data/lib/parser/current_arg_stack.rb +5 -2
  10. data/lib/parser/diagnostic.rb +1 -1
  11. data/lib/parser/diagnostic/engine.rb +1 -2
  12. data/lib/parser/lexer.rb +887 -803
  13. data/lib/parser/macruby.rb +2214 -2189
  14. data/lib/parser/max_numparam_stack.rb +13 -5
  15. data/lib/parser/messages.rb +18 -0
  16. data/lib/parser/meta.rb +6 -5
  17. data/lib/parser/ruby18.rb +9 -3
  18. data/lib/parser/ruby19.rb +2297 -2289
  19. data/lib/parser/ruby20.rb +2413 -2397
  20. data/lib/parser/ruby21.rb +2419 -2411
  21. data/lib/parser/ruby22.rb +2468 -2460
  22. data/lib/parser/ruby23.rb +2452 -2452
  23. data/lib/parser/ruby24.rb +2435 -2430
  24. data/lib/parser/ruby25.rb +2220 -2214
  25. data/lib/parser/ruby26.rb +2220 -2214
  26. data/lib/parser/ruby27.rb +3715 -3615
  27. data/lib/parser/ruby28.rb +8047 -0
  28. data/lib/parser/ruby30.rb +8060 -0
  29. data/lib/parser/ruby31.rb +8226 -0
  30. data/lib/parser/rubymotion.rb +2190 -2182
  31. data/lib/parser/runner.rb +31 -2
  32. data/lib/parser/runner/ruby_rewrite.rb +2 -2
  33. data/lib/parser/source/buffer.rb +53 -28
  34. data/lib/parser/source/comment.rb +14 -1
  35. data/lib/parser/source/comment/associator.rb +31 -8
  36. data/lib/parser/source/map/method_definition.rb +25 -0
  37. data/lib/parser/source/range.rb +10 -3
  38. data/lib/parser/source/tree_rewriter.rb +100 -10
  39. data/lib/parser/source/tree_rewriter/action.rb +114 -21
  40. data/lib/parser/static_environment.rb +4 -0
  41. data/lib/parser/tree_rewriter.rb +1 -2
  42. data/lib/parser/variables_stack.rb +4 -0
  43. data/lib/parser/version.rb +1 -1
  44. data/parser.gemspec +3 -18
  45. metadata +17 -98
  46. data/.gitignore +0 -33
  47. data/.travis.yml +0 -42
  48. data/.yardopts +0 -21
  49. data/CHANGELOG.md +0 -1075
  50. data/CONTRIBUTING.md +0 -17
  51. data/Gemfile +0 -10
  52. data/README.md +0 -309
  53. data/Rakefile +0 -166
  54. data/ci/run_rubocop_specs +0 -14
  55. data/doc/AST_FORMAT.md +0 -2180
  56. data/doc/CUSTOMIZATION.md +0 -37
  57. data/doc/INTERNALS.md +0 -21
  58. data/doc/css/.gitkeep +0 -0
  59. data/doc/css/common.css +0 -68
  60. data/lib/parser/lexer.rl +0 -2536
  61. data/lib/parser/macruby.y +0 -2198
  62. data/lib/parser/ruby18.y +0 -1934
  63. data/lib/parser/ruby19.y +0 -2175
  64. data/lib/parser/ruby20.y +0 -2353
  65. data/lib/parser/ruby21.y +0 -2357
  66. data/lib/parser/ruby22.y +0 -2364
  67. data/lib/parser/ruby23.y +0 -2370
  68. data/lib/parser/ruby24.y +0 -2408
  69. data/lib/parser/ruby25.y +0 -2405
  70. data/lib/parser/ruby26.y +0 -2413
  71. data/lib/parser/ruby27.y +0 -2941
  72. data/lib/parser/rubymotion.y +0 -2182
  73. data/test/bug_163/fixtures/input.rb +0 -5
  74. data/test/bug_163/fixtures/output.rb +0 -5
  75. data/test/bug_163/rewriter.rb +0 -20
  76. data/test/helper.rb +0 -60
  77. data/test/parse_helper.rb +0 -319
  78. data/test/racc_coverage_helper.rb +0 -133
  79. data/test/test_base.rb +0 -31
  80. data/test/test_current.rb +0 -29
  81. data/test/test_diagnostic.rb +0 -96
  82. data/test/test_diagnostic_engine.rb +0 -62
  83. data/test/test_encoding.rb +0 -99
  84. data/test/test_lexer.rb +0 -3608
  85. data/test/test_lexer_stack_state.rb +0 -78
  86. data/test/test_parse_helper.rb +0 -80
  87. data/test/test_parser.rb +0 -9430
  88. data/test/test_runner_parse.rb +0 -35
  89. data/test/test_runner_rewrite.rb +0 -47
  90. data/test/test_source_buffer.rb +0 -162
  91. data/test/test_source_comment.rb +0 -36
  92. data/test/test_source_comment_associator.rb +0 -367
  93. data/test/test_source_map.rb +0 -15
  94. data/test/test_source_range.rb +0 -187
  95. data/test/test_source_rewriter.rb +0 -541
  96. data/test/test_source_rewriter_action.rb +0 -46
  97. data/test/test_source_tree_rewriter.rb +0 -253
  98. data/test/test_static_environment.rb +0 -45
  99. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  100. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  101. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if(true)
4
- puts "Hello, world!"
5
- end
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if true
4
- puts "Hello, world!"
5
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class Rewriter < Parser::Rewriter
4
- def on_if(node)
5
- # Crude, totally-not-usable-in-the-real-world code to remove optional
6
- # parens from control keywords.
7
- #
8
- # In a perfect test scenario we'd simply make this a no-op, to demonstrate
9
- # that the bug happens when any rewriter is loaded regardless of whether it
10
- # actually changes anything but that makes assertions much harder to get
11
- # right. It's much easier to just show that the file did, or did not
12
- # get changed.
13
- if node.children[0].type == :begin
14
- replace node.children[0].loc.begin, ' '
15
- remove node.children[0].loc.end
16
- end
17
-
18
- super
19
- end
20
- end
data/test/helper.rb DELETED
@@ -1,60 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'tempfile'
4
- require 'minitest/test'
5
-
6
- require 'simplecov'
7
-
8
- if ENV.include?('COVERAGE') && SimpleCov.usable?
9
- require_relative 'racc_coverage_helper'
10
-
11
- RaccCoverage.start(
12
- %w(
13
- ruby18.y
14
- ruby19.y
15
- ruby20.y
16
- ruby21.y
17
- ruby22.y
18
- ruby23.y
19
- ruby24.y
20
- ruby25.y
21
- ruby26.y
22
- ruby27.y
23
- ),
24
- File.expand_path('../../lib/parser', __FILE__))
25
-
26
- # Report results faster.
27
- at_exit { RaccCoverage.stop }
28
-
29
- SimpleCov.start do
30
- self.formatter = SimpleCov::Formatter::MultiFormatter[
31
- SimpleCov::Formatter::HTMLFormatter,
32
- ]
33
-
34
- add_group 'Grammars' do |source_file|
35
- source_file.filename =~ %r{\.y$}
36
- end
37
-
38
- # Exclude the testsuite itself.
39
- add_filter '/test/'
40
-
41
- # Exclude generated files.
42
- add_filter do |source_file|
43
- source_file.filename =~ %r{/lib/parser/(lexer|ruby\d+|macruby|rubymotion)\.rb$}
44
- end
45
- end
46
- end
47
-
48
- # minitest/autorun must go after SimpleCov to preserve
49
- # correct order of at_exit hooks.
50
- require 'minitest/autorun'
51
-
52
- $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
53
- require 'parser'
54
-
55
- class Parser::AST::Node
56
- def initialize(type, *)
57
- raise "Type #{type} missing from Parser::Meta::NODE_TYPES" unless Parser::Meta::NODE_TYPES.include?(type)
58
- super
59
- end
60
- end
data/test/parse_helper.rb DELETED
@@ -1,319 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module ParseHelper
4
- include AST::Sexp
5
-
6
- require 'parser/all'
7
- require 'parser/macruby'
8
- require 'parser/rubymotion'
9
-
10
- ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 mac ios)
11
-
12
- def setup
13
- @diagnostics = []
14
-
15
- super if defined?(super)
16
- end
17
-
18
- def parser_for_ruby_version(version)
19
- case version
20
- when '1.8' then parser = Parser::Ruby18.new
21
- when '1.9' then parser = Parser::Ruby19.new
22
- when '2.0' then parser = Parser::Ruby20.new
23
- when '2.1' then parser = Parser::Ruby21.new
24
- when '2.2' then parser = Parser::Ruby22.new
25
- when '2.3' then parser = Parser::Ruby23.new
26
- when '2.4' then parser = Parser::Ruby24.new
27
- when '2.5' then parser = Parser::Ruby25.new
28
- when '2.6' then parser = Parser::Ruby26.new
29
- when '2.7' then parser = Parser::Ruby27.new
30
- when 'mac' then parser = Parser::MacRuby.new
31
- when 'ios' then parser = Parser::RubyMotion.new
32
- else raise "Unrecognized Ruby version #{version}"
33
- end
34
-
35
- parser.diagnostics.consumer = lambda do |diagnostic|
36
- @diagnostics << diagnostic
37
- end
38
-
39
- parser
40
- end
41
-
42
- def with_versions(versions)
43
- (versions & ALL_VERSIONS).each do |version|
44
- @diagnostics.clear
45
-
46
- parser = parser_for_ruby_version(version)
47
- yield version, parser
48
- end
49
- end
50
-
51
- def assert_source_range(begin_pos, end_pos, range, version, what)
52
- assert range.is_a?(Parser::Source::Range),
53
- "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
54
-
55
- assert_equal begin_pos, range.begin_pos,
56
- "(#{version}) begin of #{what}"
57
-
58
- assert_equal end_pos, range.end_pos,
59
- "(#{version}) end of #{what}"
60
- end
61
-
62
- # Use like this:
63
- # ~~~
64
- # assert_parses(
65
- # s(:send, s(:lit, 10), :+, s(:lit, 20))
66
- # %q{10 + 20},
67
- # %q{~~~~~~~ expression
68
- # | ^ operator
69
- # | ~~ expression (lit)
70
- # },
71
- # %w(1.8 1.9) # optional
72
- # )
73
- # ~~~
74
- def assert_parses(ast, code, source_maps='', versions=ALL_VERSIONS)
75
- with_versions(versions) do |version, parser|
76
- try_parsing(ast, code, parser, source_maps, version)
77
- end
78
-
79
- # Also try parsing with lexer set to use UTF-32LE internally
80
- with_versions(versions) do |version, parser|
81
- parser.instance_eval { @lexer.force_utf32 = true }
82
- try_parsing(ast, code, parser, source_maps, version)
83
- end
84
- end
85
-
86
- def try_parsing(ast, code, parser, source_maps, version)
87
- source_file = Parser::Source::Buffer.new('(assert_parses)')
88
- source_file.source = code
89
-
90
- begin
91
- parsed_ast = parser.parse(source_file)
92
- rescue => exc
93
- backtrace = exc.backtrace
94
- Exception.instance_method(:initialize).bind(exc).
95
- call("(#{version}) #{exc.message}")
96
- exc.set_backtrace(backtrace)
97
- raise
98
- end
99
-
100
- if ast.nil?
101
- assert_nil parsed_ast, "(#{version}) AST equality"
102
- return
103
- end
104
-
105
- assert_equal ast, parsed_ast,
106
- "(#{version}) AST equality"
107
-
108
- parse_source_map_descriptions(source_maps) \
109
- do |begin_pos, end_pos, map_field, ast_path, line|
110
-
111
- astlet = traverse_ast(parsed_ast, ast_path)
112
-
113
- if astlet.nil?
114
- # This is a testsuite bug.
115
- raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
116
- end
117
-
118
- assert astlet.frozen?
119
-
120
- assert astlet.location.respond_to?(map_field),
121
- "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
122
-
123
- range = astlet.location.send(map_field)
124
-
125
- assert_source_range(begin_pos, end_pos, range, version, line.inspect)
126
- end
127
-
128
- assert parser.instance_eval { @lexer }.cmdarg.empty?,
129
- "(#{version}) expected cmdarg to be empty after parsing"
130
-
131
- assert_equal 0, parser.instance_eval { @lexer.instance_eval { @paren_nest } },
132
- "(#{version}) expected paren_nest to be 0 after parsing"
133
- end
134
-
135
- # Use like this:
136
- # ~~~
137
- # assert_diagnoses(
138
- # [:warning, :ambiguous_prefix, { prefix: '*' }],
139
- # %q{foo *bar},
140
- # %q{ ^ location
141
- # | ~~~ highlights (0)})
142
- # ~~~
143
- def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
144
- with_versions(versions) do |version, parser|
145
- source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
146
- source_file.source = code
147
-
148
- begin
149
- parser = parser.parse(source_file)
150
- rescue Parser::SyntaxError
151
- # do nothing; the diagnostic was reported
152
- end
153
-
154
- assert_equal 1, @diagnostics.count,
155
- "(#{version}) emits a single diagnostic, not\n" \
156
- "#{@diagnostics.map(&:render).join("\n")}"
157
-
158
- emitted_diagnostic = @diagnostics.first
159
-
160
- level, reason, arguments = diagnostic
161
- arguments ||= {}
162
- message = Parser::MESSAGES[reason] % arguments
163
-
164
- assert_equal level, emitted_diagnostic.level
165
- assert_equal reason, emitted_diagnostic.reason
166
- assert_equal arguments, emitted_diagnostic.arguments
167
- assert_equal message, emitted_diagnostic.message
168
-
169
- parse_source_map_descriptions(source_maps) \
170
- do |begin_pos, end_pos, map_field, ast_path, line|
171
-
172
- case map_field
173
- when 'location'
174
- assert_source_range begin_pos, end_pos,
175
- emitted_diagnostic.location,
176
- version, 'location'
177
-
178
- when 'highlights'
179
- index = ast_path.first.to_i
180
-
181
- assert_source_range begin_pos, end_pos,
182
- emitted_diagnostic.highlights[index],
183
- version, "#{index}th highlight"
184
-
185
- else
186
- raise "Unknown diagnostic range #{map_field}"
187
- end
188
- end
189
- end
190
- end
191
-
192
- # Use like this:
193
- # ~~~
194
- # assert_diagnoses_many(
195
- # [
196
- # [:warning, :ambiguous_literal],
197
- # [:error, :unexpected_token, { :token => :tLCURLY }]
198
- # ],
199
- # %q{m /foo/ {}},
200
- # SINCE_2_4)
201
- # ~~~
202
- def assert_diagnoses_many(diagnostics, code, versions=ALL_VERSIONS)
203
- with_versions(versions) do |version, parser|
204
- source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)')
205
- source_file.source = code
206
-
207
- begin
208
- parser = parser.parse(source_file)
209
- rescue Parser::SyntaxError
210
- # do nothing; the diagnostic was reported
211
- end
212
-
213
- assert_equal diagnostics.count, @diagnostics.count
214
-
215
- diagnostics.zip(@diagnostics) do |expected_diagnostic, actual_diagnostic|
216
- level, reason, arguments = expected_diagnostic
217
- arguments ||= {}
218
- message = Parser::MESSAGES[reason] % arguments
219
-
220
- assert_equal level, actual_diagnostic.level
221
- assert_equal reason, actual_diagnostic.reason
222
- assert_equal arguments, actual_diagnostic.arguments
223
- assert_equal message, actual_diagnostic.message
224
- end
225
- end
226
- end
227
-
228
- def refute_diagnoses(code, versions=ALL_VERSIONS)
229
- with_versions(versions) do |version, parser|
230
- source_file = Parser::Source::Buffer.new('(refute_diagnoses)')
231
- source_file.source = code
232
-
233
- begin
234
- parser = parser.parse(source_file)
235
- rescue Parser::SyntaxError
236
- # do nothing; the diagnostic was reported
237
- end
238
-
239
- assert_empty @diagnostics,
240
- "(#{version}) emits no diagnostics, not\n" \
241
- "#{@diagnostics.map(&:render).join("\n")}"
242
- end
243
- end
244
-
245
- def assert_context(context, code, versions=ALL_VERSIONS)
246
- with_versions(versions) do |version, parser|
247
- source_file = Parser::Source::Buffer.new('(assert_context)')
248
- source_file.source = code
249
-
250
- begin
251
- parser.parse(source_file)
252
- rescue Parser::SyntaxError
253
- # do nothing; the diagnostic was reported
254
- end
255
-
256
- assert_equal parser.context.stack, context, "(#{version}) parsing context"
257
- end
258
- end
259
-
260
- SOURCE_MAP_DESCRIPTION_RE =
261
- /(?x)
262
- ^(?# $1 skip) ^(\s*)
263
- (?# $2 highlight) ([~\^]+)
264
- \s+
265
- (?# $3 source_map_field) ([a-z_]+)
266
- (?# $5 ast_path) (\s+\(([a-z_.\/0-9]+)\))?
267
- $/
268
-
269
- def parse_source_map_descriptions(descriptions)
270
- unless block_given?
271
- return to_enum(:parse_source_map_descriptions, descriptions)
272
- end
273
-
274
- descriptions.each_line do |line|
275
- # Remove leading " |", if it exists.
276
- line = line.sub(/^\s*\|/, '').rstrip
277
-
278
- next if line.empty?
279
-
280
- if (match = SOURCE_MAP_DESCRIPTION_RE.match(line))
281
- begin_pos = match[1].length
282
- end_pos = begin_pos + match[2].length
283
- source_map_field = match[3]
284
-
285
- if match[5]
286
- ast_path = match[5].split('.')
287
- else
288
- ast_path = []
289
- end
290
-
291
- yield begin_pos, end_pos, source_map_field, ast_path, line
292
- else
293
- raise "Cannot parse source map description line: #{line.inspect}."
294
- end
295
- end
296
- end
297
-
298
- def traverse_ast(ast, path)
299
- path.inject(ast) do |astlet, path_component|
300
- # Split "dstr/2" to :dstr and 1
301
- type_str, index_str = path_component.split('/')
302
-
303
- type = type_str.to_sym
304
-
305
- if index_str.nil?
306
- index = 0
307
- else
308
- index = index_str.to_i - 1
309
- end
310
-
311
- matching_children = \
312
- astlet.children.select do |child|
313
- AST::Node === child && child.type == type
314
- end
315
-
316
- matching_children[index]
317
- end
318
- end
319
- end
@@ -1,133 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'racc/grammarfileparser'
4
-
5
- # Unfortunately, Ruby's Coverage module ignores module_eval statements,
6
- # which Racc uses to map `parser.y` locations in the generated
7
- # `parser.rb`.
8
- module RaccCoverage
9
- @coverage = {}
10
- @base_path = nil
11
- @trace = nil
12
-
13
- def self.start(parsers, base_path)
14
- @base_path = base_path
15
-
16
- parsers.each do |parser|
17
- @coverage[parser] = extract_interesting_lines(parser, base_path)
18
- end
19
-
20
- @trace = TracePoint.new(:line) do |trace|
21
- lineno = trace.lineno - 1
22
-
23
- if (line_coverage = @coverage[trace.path])
24
- if line_coverage[lineno]
25
- line_coverage[lineno] += 1
26
- end
27
- end
28
- end
29
- @trace.enable
30
- end
31
-
32
- def self.stop
33
- @trace.disable
34
- end
35
-
36
- # Ruby's TracePoint#lineno will point only on "interesting" lines,
37
- # i.e.: only code (no comments or empty lines), no `end` keywords,
38
- # and for multi-line statements, only the first line of the statement.
39
- #
40
- # This method implements a very dumb Ruby parser, which skips empty lines
41
- # or lines with just comments, `end` keywords, and correctly handles
42
- # multi-line statements of the following form:
43
- #
44
- # * All lines of the statement except the last must end with `,`, `.` or `(`.
45
- #
46
- # Coverage can be disabled for code regions with annotations :nocov: and :cov:.
47
- #
48
- # Also, for best results, all actions should be delimited by at least
49
- # one non-action line.
50
- #
51
- def self.extract_interesting_lines(parser, base_path)
52
- grammar_source = File.join(@base_path, parser)
53
- grammar_file = Racc::GrammarFileParser.parse_file(grammar_source)
54
-
55
- ruby_sources = [
56
- # Header and footer aren't passed through module_eval
57
- # in Racc-generated file, so the location info is lost.
58
- *grammar_file.params.inner,
59
- ].compact
60
-
61
- grammar_file.grammar.each_rule do |rule|
62
- source = rule.action.source
63
- next if source.nil?
64
-
65
- ruby_sources << source
66
- end
67
-
68
- lines = []
69
-
70
- ruby_sources.each do |source|
71
- first_line = source.lineno
72
-
73
- state = :first_line
74
-
75
- source.text.each_line.with_index do |line, index|
76
- line = line.strip
77
-
78
- continues = line.end_with?(',') ||
79
- line.end_with?('(') ||
80
- line.end_with?('.')
81
-
82
- case state
83
- when :first_line
84
- if line =~ /:nocov/
85
- state = :nocov
86
- next
87
- elsif line.empty? ||
88
- line == 'end' ||
89
- line.start_with?('#')
90
- next
91
- elsif continues
92
- state = :mid_line
93
- end
94
-
95
- lines[first_line + index - 1] = 0
96
-
97
- when :mid_line
98
- unless continues
99
- state = :first_line
100
- end
101
-
102
- when :nocov
103
- if line =~ /:cov:/
104
- state = :first_line
105
- end
106
- end
107
- end
108
- end
109
-
110
- lines
111
- end
112
-
113
- def self.result
114
- result =
115
- @coverage.map do |parser, coverage|
116
- [File.join(@base_path, parser), coverage]
117
- end
118
-
119
- Hash[result]
120
- end
121
- end
122
-
123
- class << SimpleCov
124
- def result_with_racc_coverage
125
- @result ||= SimpleCov::Result.new(
126
- Coverage.result.merge(RaccCoverage.result))
127
-
128
- result_without_racc_coverage
129
- end
130
-
131
- alias result_without_racc_coverage result
132
- alias result result_with_racc_coverage
133
- end