parser 2.5.1.0 → 3.0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/lib/parser.rb +4 -0
  3. data/lib/parser/all.rb +3 -0
  4. data/lib/parser/ast/processor.rb +49 -1
  5. data/lib/parser/base.rb +30 -6
  6. data/lib/parser/builders/default.rb +586 -29
  7. data/lib/parser/context.rb +17 -0
  8. data/lib/parser/current.rb +34 -7
  9. data/lib/parser/current_arg_stack.rb +46 -0
  10. data/lib/parser/diagnostic.rb +1 -1
  11. data/lib/parser/diagnostic/engine.rb +1 -2
  12. data/lib/parser/lexer.rb +23780 -0
  13. data/lib/parser/lexer/dedenter.rb +52 -49
  14. data/lib/parser/lexer/literal.rb +4 -0
  15. data/lib/parser/lexer/stack_state.rb +4 -0
  16. data/lib/parser/macruby.rb +6149 -0
  17. data/lib/parser/max_numparam_stack.rb +56 -0
  18. data/lib/parser/messages.rb +74 -44
  19. data/lib/parser/meta.rb +13 -3
  20. data/lib/parser/ruby18.rb +5667 -0
  21. data/lib/parser/ruby19.rb +6092 -0
  22. data/lib/parser/ruby20.rb +6527 -0
  23. data/lib/parser/ruby21.rb +6578 -0
  24. data/lib/parser/ruby22.rb +6613 -0
  25. data/lib/parser/ruby23.rb +6624 -0
  26. data/lib/parser/ruby24.rb +6694 -0
  27. data/lib/parser/ruby25.rb +6662 -0
  28. data/lib/parser/ruby26.rb +6676 -0
  29. data/lib/parser/ruby27.rb +7862 -0
  30. data/lib/parser/ruby28.rb +8047 -0
  31. data/lib/parser/ruby30.rb +8060 -0
  32. data/lib/parser/ruby31.rb +8075 -0
  33. data/lib/parser/rubymotion.rb +6086 -0
  34. data/lib/parser/runner.rb +36 -2
  35. data/lib/parser/runner/ruby_parse.rb +2 -2
  36. data/lib/parser/runner/ruby_rewrite.rb +2 -2
  37. data/lib/parser/source/buffer.rb +54 -29
  38. data/lib/parser/source/comment.rb +18 -5
  39. data/lib/parser/source/comment/associator.rb +34 -11
  40. data/lib/parser/source/map.rb +1 -1
  41. data/lib/parser/source/map/method_definition.rb +25 -0
  42. data/lib/parser/source/range.rb +20 -4
  43. data/lib/parser/source/tree_rewriter.rb +146 -16
  44. data/lib/parser/source/tree_rewriter/action.rb +137 -28
  45. data/lib/parser/static_environment.rb +14 -0
  46. data/lib/parser/tree_rewriter.rb +3 -3
  47. data/lib/parser/variables_stack.rb +36 -0
  48. data/lib/parser/version.rb +1 -1
  49. data/parser.gemspec +13 -21
  50. metadata +34 -98
  51. data/.gitignore +0 -32
  52. data/.travis.yml +0 -21
  53. data/.yardopts +0 -21
  54. data/CHANGELOG.md +0 -909
  55. data/CONTRIBUTING.md +0 -17
  56. data/Gemfile +0 -10
  57. data/README.md +0 -301
  58. data/Rakefile +0 -165
  59. data/doc/AST_FORMAT.md +0 -1718
  60. data/doc/CUSTOMIZATION.md +0 -37
  61. data/doc/INTERNALS.md +0 -21
  62. data/doc/css/.gitkeep +0 -0
  63. data/doc/css/common.css +0 -68
  64. data/lib/parser/lexer.rl +0 -2376
  65. data/lib/parser/macruby.y +0 -2198
  66. data/lib/parser/ruby18.y +0 -1934
  67. data/lib/parser/ruby19.y +0 -2175
  68. data/lib/parser/ruby20.y +0 -2353
  69. data/lib/parser/ruby21.y +0 -2357
  70. data/lib/parser/ruby22.y +0 -2364
  71. data/lib/parser/ruby23.y +0 -2370
  72. data/lib/parser/ruby24.y +0 -2395
  73. data/lib/parser/ruby25.y +0 -2392
  74. data/lib/parser/ruby26.y +0 -2392
  75. data/lib/parser/rubymotion.y +0 -2182
  76. data/test/bug_163/fixtures/input.rb +0 -5
  77. data/test/bug_163/fixtures/output.rb +0 -5
  78. data/test/bug_163/rewriter.rb +0 -20
  79. data/test/helper.rb +0 -52
  80. data/test/parse_helper.rb +0 -315
  81. data/test/racc_coverage_helper.rb +0 -133
  82. data/test/test_base.rb +0 -31
  83. data/test/test_current.rb +0 -27
  84. data/test/test_diagnostic.rb +0 -96
  85. data/test/test_diagnostic_engine.rb +0 -62
  86. data/test/test_encoding.rb +0 -99
  87. data/test/test_lexer.rb +0 -3537
  88. data/test/test_lexer_stack_state.rb +0 -78
  89. data/test/test_parse_helper.rb +0 -80
  90. data/test/test_parser.rb +0 -6968
  91. data/test/test_runner_rewrite.rb +0 -47
  92. data/test/test_source_buffer.rb +0 -162
  93. data/test/test_source_comment.rb +0 -36
  94. data/test/test_source_comment_associator.rb +0 -367
  95. data/test/test_source_map.rb +0 -15
  96. data/test/test_source_range.rb +0 -172
  97. data/test/test_source_rewriter.rb +0 -541
  98. data/test/test_source_rewriter_action.rb +0 -46
  99. data/test/test_source_tree_rewriter.rb +0 -173
  100. data/test/test_static_environment.rb +0 -45
  101. data/test/using_tree_rewriter/fixtures/input.rb +0 -3
  102. data/test/using_tree_rewriter/fixtures/output.rb +0 -3
  103. data/test/using_tree_rewriter/using_tree_rewriter.rb +0 -9
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if(true)
4
- puts "Hello, world!"
5
- end
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- if true
4
- puts "Hello, world!"
5
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class Rewriter < Parser::Rewriter
4
- def on_if(node)
5
- # Crude, totally-not-usable-in-the-real-world code to remove optional
6
- # parens from control keywords.
7
- #
8
- # In a perfect test scenario we'd simply make this a no-op, to demonstrate
9
- # that the bug happens when any rewriter is loaded regardless of whether it
10
- # actually changes anything but that makes assertions much harder to get
11
- # right. It's much easier to just show that the file did, or did not
12
- # get changed.
13
- if node.children[0].type == :begin
14
- replace node.children[0].loc.begin, ' '
15
- remove node.children[0].loc.end
16
- end
17
-
18
- super
19
- end
20
- end
data/test/helper.rb DELETED
@@ -1,52 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'tempfile'
4
- require 'minitest/test'
5
-
6
- require 'simplecov'
7
-
8
- if ENV.include?('COVERAGE') && SimpleCov.usable?
9
- require_relative 'racc_coverage_helper'
10
-
11
- RaccCoverage.start(
12
- %w(
13
- ruby18.y
14
- ruby19.y
15
- ruby20.y
16
- ruby21.y
17
- ruby22.y
18
- ruby23.y
19
- ruby24.y
20
- ruby25.y
21
- ruby26.y
22
- ),
23
- File.expand_path('../../lib/parser', __FILE__))
24
-
25
- # Report results faster.
26
- at_exit { RaccCoverage.stop }
27
-
28
- SimpleCov.start do
29
- self.formatter = SimpleCov::Formatter::MultiFormatter[
30
- SimpleCov::Formatter::HTMLFormatter,
31
- ]
32
-
33
- add_group 'Grammars' do |source_file|
34
- source_file.filename =~ %r{\.y$}
35
- end
36
-
37
- # Exclude the testsuite itself.
38
- add_filter '/test/'
39
-
40
- # Exclude generated files.
41
- add_filter do |source_file|
42
- source_file.filename =~ %r{/lib/parser/(lexer|ruby\d+|macruby|rubymotion)\.rb$}
43
- end
44
- end
45
- end
46
-
47
- # minitest/autorun must go after SimpleCov to preserve
48
- # correct order of at_exit hooks.
49
- require 'minitest/autorun'
50
-
51
- $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
52
- require 'parser'
data/test/parse_helper.rb DELETED
@@ -1,315 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module ParseHelper
4
- include AST::Sexp
5
-
6
- require 'parser/all'
7
- require 'parser/macruby'
8
- require 'parser/rubymotion'
9
-
10
- ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 mac ios)
11
-
12
- def setup
13
- @diagnostics = []
14
-
15
- super if defined?(super)
16
- end
17
-
18
- def parser_for_ruby_version(version)
19
- case version
20
- when '1.8' then parser = Parser::Ruby18.new
21
- when '1.9' then parser = Parser::Ruby19.new
22
- when '2.0' then parser = Parser::Ruby20.new
23
- when '2.1' then parser = Parser::Ruby21.new
24
- when '2.2' then parser = Parser::Ruby22.new
25
- when '2.3' then parser = Parser::Ruby23.new
26
- when '2.4' then parser = Parser::Ruby24.new
27
- when '2.5' then parser = Parser::Ruby25.new
28
- when '2.6' then parser = Parser::Ruby26.new
29
- when 'mac' then parser = Parser::MacRuby.new
30
- when 'ios' then parser = Parser::RubyMotion.new
31
- else raise "Unrecognized Ruby version #{version}"
32
- end
33
-
34
- parser.diagnostics.consumer = lambda do |diagnostic|
35
- @diagnostics << diagnostic
36
- end
37
-
38
- parser
39
- end
40
-
41
- def with_versions(versions)
42
- (versions & ALL_VERSIONS).each do |version|
43
- @diagnostics.clear
44
-
45
- parser = parser_for_ruby_version(version)
46
- yield version, parser
47
- end
48
- end
49
-
50
- def assert_source_range(begin_pos, end_pos, range, version, what)
51
- assert range.is_a?(Parser::Source::Range),
52
- "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
53
-
54
- assert_equal begin_pos, range.begin_pos,
55
- "(#{version}) begin of #{what}"
56
-
57
- assert_equal end_pos, range.end_pos,
58
- "(#{version}) end of #{what}"
59
- end
60
-
61
- # Use like this:
62
- # ~~~
63
- # assert_parses(
64
- # s(:send, s(:lit, 10), :+, s(:lit, 20))
65
- # %q{10 + 20},
66
- # %q{~~~~~~~ expression
67
- # | ^ operator
68
- # | ~~ expression (lit)
69
- # },
70
- # %w(1.8 1.9) # optional
71
- # )
72
- # ~~~
73
- def assert_parses(ast, code, source_maps='', versions=ALL_VERSIONS)
74
- with_versions(versions) do |version, parser|
75
- try_parsing(ast, code, parser, source_maps, version)
76
- end
77
-
78
- # Also try parsing with lexer set to use UTF-32LE internally
79
- with_versions(versions) do |version, parser|
80
- parser.instance_eval { @lexer.force_utf32 = true }
81
- try_parsing(ast, code, parser, source_maps, version)
82
- end
83
- end
84
-
85
- def try_parsing(ast, code, parser, source_maps, version)
86
- source_file = Parser::Source::Buffer.new('(assert_parses)')
87
- source_file.source = code
88
-
89
- begin
90
- parsed_ast = parser.parse(source_file)
91
- rescue => exc
92
- backtrace = exc.backtrace
93
- Exception.instance_method(:initialize).bind(exc).
94
- call("(#{version}) #{exc.message}")
95
- exc.set_backtrace(backtrace)
96
- raise
97
- end
98
-
99
- if ast.nil?
100
- assert_nil parsed_ast, "(#{version}) AST equality"
101
- return
102
- end
103
-
104
- assert_equal ast, parsed_ast,
105
- "(#{version}) AST equality"
106
-
107
- parse_source_map_descriptions(source_maps) \
108
- do |begin_pos, end_pos, map_field, ast_path, line|
109
-
110
- astlet = traverse_ast(parsed_ast, ast_path)
111
-
112
- if astlet.nil?
113
- # This is a testsuite bug.
114
- raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
115
- end
116
-
117
- assert astlet.frozen?
118
-
119
- assert astlet.location.respond_to?(map_field),
120
- "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
121
-
122
- range = astlet.location.send(map_field)
123
-
124
- assert_source_range(begin_pos, end_pos, range, version, line.inspect)
125
- end
126
-
127
- assert_equal parser.instance_eval { @lexer }.cmdarg.instance_eval { @stack }, 0,
128
- "(#{version}) expected cmdarg to be empty after parsing"
129
- end
130
-
131
- # Use like this:
132
- # ~~~
133
- # assert_diagnoses(
134
- # [:warning, :ambiguous_prefix, { prefix: '*' }],
135
- # %q{foo *bar},
136
- # %q{ ^ location
137
- # | ~~~ highlights (0)})
138
- # ~~~
139
- def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
140
- with_versions(versions) do |version, parser|
141
- source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
142
- source_file.source = code
143
-
144
- begin
145
- parser = parser.parse(source_file)
146
- rescue Parser::SyntaxError
147
- # do nothing; the diagnostic was reported
148
- end
149
-
150
- assert_equal 1, @diagnostics.count,
151
- "(#{version}) emits a single diagnostic, not\n" \
152
- "#{@diagnostics.map(&:render).join("\n")}"
153
-
154
- emitted_diagnostic = @diagnostics.first
155
-
156
- level, reason, arguments = diagnostic
157
- arguments ||= {}
158
- message = Parser::MESSAGES[reason] % arguments
159
-
160
- assert_equal level, emitted_diagnostic.level
161
- assert_equal reason, emitted_diagnostic.reason
162
- assert_equal arguments, emitted_diagnostic.arguments
163
- assert_equal message, emitted_diagnostic.message
164
-
165
- parse_source_map_descriptions(source_maps) \
166
- do |begin_pos, end_pos, map_field, ast_path, line|
167
-
168
- case map_field
169
- when 'location'
170
- assert_source_range begin_pos, end_pos,
171
- emitted_diagnostic.location,
172
- version, 'location'
173
-
174
- when 'highlights'
175
- index = ast_path.first.to_i
176
-
177
- assert_source_range begin_pos, end_pos,
178
- emitted_diagnostic.highlights[index],
179
- version, "#{index}th highlight"
180
-
181
- else
182
- raise "Unknown diagnostic range #{map_field}"
183
- end
184
- end
185
- end
186
- end
187
-
188
- # Use like this:
189
- # ~~~
190
- # assert_diagnoses_many(
191
- # [
192
- # [:warning, :ambiguous_literal],
193
- # [:error, :unexpected_token, { :token => :tLCURLY }]
194
- # ],
195
- # %q{m /foo/ {}},
196
- # SINCE_2_4)
197
- # ~~~
198
- def assert_diagnoses_many(diagnostics, code, versions=ALL_VERSIONS)
199
- with_versions(versions) do |version, parser|
200
- source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)')
201
- source_file.source = code
202
-
203
- begin
204
- parser = parser.parse(source_file)
205
- rescue Parser::SyntaxError
206
- # do nothing; the diagnostic was reported
207
- end
208
-
209
- assert_equal diagnostics.count, @diagnostics.count
210
-
211
- diagnostics.zip(@diagnostics) do |expected_diagnostic, actual_diagnostic|
212
- level, reason, arguments = expected_diagnostic
213
- arguments ||= {}
214
- message = Parser::MESSAGES[reason] % arguments
215
-
216
- assert_equal level, actual_diagnostic.level
217
- assert_equal reason, actual_diagnostic.reason
218
- assert_equal arguments, actual_diagnostic.arguments
219
- assert_equal message, actual_diagnostic.message
220
- end
221
- end
222
- end
223
-
224
- def refute_diagnoses(code, versions=ALL_VERSIONS)
225
- with_versions(versions) do |version, parser|
226
- source_file = Parser::Source::Buffer.new('(refute_diagnoses)')
227
- source_file.source = code
228
-
229
- begin
230
- parser = parser.parse(source_file)
231
- rescue Parser::SyntaxError
232
- # do nothing; the diagnostic was reported
233
- end
234
-
235
- assert_empty @diagnostics,
236
- "(#{version}) emits no diagnostics, not\n" \
237
- "#{@diagnostics.map(&:render).join("\n")}"
238
- end
239
- end
240
-
241
- def assert_context(context, code, versions=ALL_VERSIONS)
242
- with_versions(versions) do |version, parser|
243
- source_file = Parser::Source::Buffer.new('(assert_context)')
244
- source_file.source = code
245
-
246
- begin
247
- parser.parse(source_file)
248
- rescue Parser::SyntaxError
249
- # do nothing; the diagnostic was reported
250
- end
251
-
252
- assert_equal parser.context.stack, context, "(#{version}) parsing context"
253
- end
254
- end
255
-
256
- SOURCE_MAP_DESCRIPTION_RE =
257
- /(?x)
258
- ^(?# $1 skip) ^(\s*)
259
- (?# $2 highlight) ([~\^]+)
260
- \s+
261
- (?# $3 source_map_field) ([a-z_]+)
262
- (?# $5 ast_path) (\s+\(([a-z_.\/0-9]+)\))?
263
- $/
264
-
265
- def parse_source_map_descriptions(descriptions)
266
- unless block_given?
267
- return to_enum(:parse_source_map_descriptions, descriptions)
268
- end
269
-
270
- descriptions.each_line do |line|
271
- # Remove leading " |", if it exists.
272
- line = line.sub(/^\s*\|/, '').rstrip
273
-
274
- next if line.empty?
275
-
276
- if (match = SOURCE_MAP_DESCRIPTION_RE.match(line))
277
- begin_pos = match[1].length
278
- end_pos = begin_pos + match[2].length
279
- source_map_field = match[3]
280
-
281
- if match[5]
282
- ast_path = match[5].split('.')
283
- else
284
- ast_path = []
285
- end
286
-
287
- yield begin_pos, end_pos, source_map_field, ast_path, line
288
- else
289
- raise "Cannot parse source map description line: #{line.inspect}."
290
- end
291
- end
292
- end
293
-
294
- def traverse_ast(ast, path)
295
- path.inject(ast) do |astlet, path_component|
296
- # Split "dstr/2" to :dstr and 1
297
- type_str, index_str = path_component.split('/')
298
-
299
- type = type_str.to_sym
300
-
301
- if index_str.nil?
302
- index = 0
303
- else
304
- index = index_str.to_i - 1
305
- end
306
-
307
- matching_children = \
308
- astlet.children.select do |child|
309
- AST::Node === child && child.type == type
310
- end
311
-
312
- matching_children[index]
313
- end
314
- end
315
- end
@@ -1,133 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'racc/grammarfileparser'
4
-
5
- # Unfortunately, Ruby's Coverage module ignores module_eval statements,
6
- # which Racc uses to map `parser.y` locations in the generated
7
- # `parser.rb`.
8
- module RaccCoverage
9
- @coverage = {}
10
- @base_path = nil
11
- @trace = nil
12
-
13
- def self.start(parsers, base_path)
14
- @base_path = base_path
15
-
16
- parsers.each do |parser|
17
- @coverage[parser] = extract_interesting_lines(parser, base_path)
18
- end
19
-
20
- @trace = TracePoint.new(:line) do |trace|
21
- lineno = trace.lineno - 1
22
-
23
- if (line_coverage = @coverage[trace.path])
24
- if line_coverage[lineno]
25
- line_coverage[lineno] += 1
26
- end
27
- end
28
- end
29
- @trace.enable
30
- end
31
-
32
- def self.stop
33
- @trace.disable
34
- end
35
-
36
- # Ruby's TracePoint#lineno will point only on "interesting" lines,
37
- # i.e.: only code (no comments or empty lines), no `end` keywords,
38
- # and for multi-line statements, only the first line of the statement.
39
- #
40
- # This method implements a very dumb Ruby parser, which skips empty lines
41
- # or lines with just comments, `end` keywords, and correctly handles
42
- # multi-line statements of the following form:
43
- #
44
- # * All lines of the statement except the last must end with `,`, `.` or `(`.
45
- #
46
- # Coverage can be disabled for code regions with annotations :nocov: and :cov:.
47
- #
48
- # Also, for best results, all actions should be delimited by at least
49
- # one non-action line.
50
- #
51
- def self.extract_interesting_lines(parser, base_path)
52
- grammar_source = File.join(@base_path, parser)
53
- grammar_file = Racc::GrammarFileParser.parse_file(grammar_source)
54
-
55
- ruby_sources = [
56
- # Header and footer aren't passed through module_eval
57
- # in Racc-generated file, so the location info is lost.
58
- *grammar_file.params.inner,
59
- ].compact
60
-
61
- grammar_file.grammar.each_rule do |rule|
62
- source = rule.action.source
63
- next if source.nil?
64
-
65
- ruby_sources << source
66
- end
67
-
68
- lines = []
69
-
70
- ruby_sources.each do |source|
71
- first_line = source.lineno
72
-
73
- state = :first_line
74
-
75
- source.text.each_line.with_index do |line, index|
76
- line = line.strip
77
-
78
- continues = line.end_with?(',') ||
79
- line.end_with?('(') ||
80
- line.end_with?('.')
81
-
82
- case state
83
- when :first_line
84
- if line =~ /:nocov/
85
- state = :nocov
86
- next
87
- elsif line.empty? ||
88
- line == 'end' ||
89
- line.start_with?('#')
90
- next
91
- elsif continues
92
- state = :mid_line
93
- end
94
-
95
- lines[first_line + index - 1] = 0
96
-
97
- when :mid_line
98
- unless continues
99
- state = :first_line
100
- end
101
-
102
- when :nocov
103
- if line =~ /:cov:/
104
- state = :first_line
105
- end
106
- end
107
- end
108
- end
109
-
110
- lines
111
- end
112
-
113
- def self.result
114
- result =
115
- @coverage.map do |parser, coverage|
116
- [File.join(@base_path, parser), coverage]
117
- end
118
-
119
- Hash[result]
120
- end
121
- end
122
-
123
- class << SimpleCov
124
- def result_with_racc_coverage
125
- @result ||= SimpleCov::Result.new(
126
- Coverage.result.merge(RaccCoverage.result))
127
-
128
- result_without_racc_coverage
129
- end
130
-
131
- alias result_without_racc_coverage result
132
- alias result result_with_racc_coverage
133
- end