parser 2.7.0.5 → 2.7.1.4

data/lib/parser/tree_rewriter.rb

@@ -28,8 +28,7 @@ module Parser
   #     EOF
   #
   #     ast           = Parser::CurrentRuby.parse code
-  #     buffer        = Parser::Source::Buffer.new('(example)')
-  #     buffer.source = code
+  #     buffer        = Parser::Source::Buffer.new('(example)', source: code)
   #     rewriter      = RemoveDo.new
   #
   #     # Rewrite the AST, returns a String with the new form.

data/lib/parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parser
-  VERSION = '2.7.0.5'
+  VERSION = '2.7.1.4'
 end

data/parser.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |spec|
                           lib/parser/ruby25.rb
                           lib/parser/ruby26.rb
                           lib/parser/ruby27.rb
+                          lib/parser/ruby28.rb
                           lib/parser/macruby.rb
                           lib/parser/rubymotion.rb
                        )
@@ -41,10 +42,10 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_dependency             'ast',       '~> 2.4.0'
+  spec.add_dependency             'ast',       '~> 2.4.1'
 
   spec.add_development_dependency 'bundler',   '>= 1.15', '< 3.0.0'
-  spec.add_development_dependency 'rake',      '~> 10.0'
+  spec.add_development_dependency 'rake',      '~> 13.0.1'
   spec.add_development_dependency 'racc',      '= 1.4.15'
   spec.add_development_dependency 'cliver',    '~> 0.3.2'
 

data/test/helper.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require 'tempfile'
-require 'minitest/test'
-
 require 'simplecov'
 
 if ENV.include?('COVERAGE') && SimpleCov.usable?
@@ -20,6 +18,7 @@ if ENV.include?('COVERAGE') && SimpleCov.usable?
       ruby25.y
       ruby26.y
       ruby27.y
+      ruby28.y
     ),
     File.expand_path('../../lib/parser', __FILE__))
 
@@ -27,9 +26,9 @@ if ENV.include?('COVERAGE') && SimpleCov.usable?
   at_exit { RaccCoverage.stop }
 
   SimpleCov.start do
-    self.formatter = SimpleCov::Formatter::MultiFormatter[
-      SimpleCov::Formatter::HTMLFormatter,
-    ]
+    self.formatter = SimpleCov::Formatter::MultiFormatter.new(
+      SimpleCov::Formatter::HTMLFormatter
+    )
 
     add_group 'Grammars' do |source_file|
       source_file.filename =~ %r{\.y$}
@@ -52,9 +51,53 @@ require 'minitest/autorun'
 $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
 require 'parser'
 
+module NodeCollector
+  extend self
+  attr_accessor :callbacks, :nodes
+  self.callbacks = []
+  self.nodes = []
+
+  def check
+    @callbacks.each do |callback|
+      @nodes.each { |node| callback.call(node) }
+    end
+    puts "#{callbacks.size} additional tests on #{nodes.size} nodes ran successfully"
+  end
+
+  Minitest.after_run { check }
+end
+
+def for_each_node(&block)
+  NodeCollector.callbacks << block
+end
+
 class Parser::AST::Node
   def initialize(type, *)
-    raise "Type #{type} missing from Parser::Meta::NODE_TYPES" unless Parser::Meta::NODE_TYPES.include?(type)
+    NodeCollector.nodes << self
     super
   end
 end
+
+# Special test extension that records a context of the parser
+# for any node that is created
+module NodeContextExt
+  module NodeExt
+    attr_reader :context
+
+    def assign_properties(properties)
+      super
+
+      if (context = properties[:context])
+        @context = context
+      end
+    end
+  end
+  Parser::AST::Node.prepend(NodeExt)
+
+  module BuilderExt
+    def n(type, children, source_map)
+      super.updated(nil, nil, context: @parser.context.stack.dup)
+    end
+  end
+  Parser::Builders::Default.prepend(BuilderExt)
+end

data/test/parse_helper.rb

@@ -7,7 +7,7 @@ module ParseHelper
   require 'parser/macruby'
   require 'parser/rubymotion'
 
-  ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 mac ios)
+  ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 mac ios)
 
   def setup
     @diagnostics = []
@@ -27,6 +27,7 @@ module ParseHelper
     when '2.5' then parser = Parser::Ruby25.new
     when '2.6' then parser = Parser::Ruby26.new
     when '2.7' then parser = Parser::Ruby27.new
+    when '2.8' then parser = Parser::Ruby28.new
     when 'mac' then parser = Parser::MacRuby.new
     when 'ios' then parser = Parser::RubyMotion.new
     else raise "Unrecognized Ruby version #{version}"
@@ -84,8 +85,7 @@ module ParseHelper
   end
 
   def try_parsing(ast, code, parser, source_maps, version)
-    source_file = Parser::Source::Buffer.new('(assert_parses)')
-    source_file.source = code
+    source_file = Parser::Source::Buffer.new('(assert_parses)', source: code)
 
     begin
       parsed_ast = parser.parse(source_file)
@@ -105,8 +105,7 @@ module ParseHelper
     assert_equal ast, parsed_ast,
                  "(#{version}) AST equality"
 
-    parse_source_map_descriptions(source_maps) \
-        do |begin_pos, end_pos, map_field, ast_path, line|
+    parse_source_map_descriptions(source_maps) do |begin_pos, end_pos, map_field, ast_path, line|
 
       astlet = traverse_ast(parsed_ast, ast_path)
 
@@ -142,8 +141,7 @@ module ParseHelper
   # ~~~
   def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_diagnoses)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -159,15 +157,14 @@ module ParseHelper
 
       level, reason, arguments = diagnostic
       arguments ||= {}
-      message     = Parser::MESSAGES[reason] % arguments
+      message     = Parser::Messages.compile(reason, arguments)
 
       assert_equal level, emitted_diagnostic.level
       assert_equal reason, emitted_diagnostic.reason
       assert_equal arguments, emitted_diagnostic.arguments
       assert_equal message, emitted_diagnostic.message
 
-      parse_source_map_descriptions(source_maps) \
-          do |begin_pos, end_pos, map_field, ast_path, line|
+      parse_source_map_descriptions(source_maps) do |begin_pos, end_pos, map_field, ast_path, line|
 
         case map_field
         when 'location'
@@ -201,8 +198,7 @@ module ParseHelper
   # ~~~
   def assert_diagnoses_many(diagnostics, code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -215,7 +211,7 @@ module ParseHelper
       diagnostics.zip(@diagnostics) do |expected_diagnostic, actual_diagnostic|
         level, reason, arguments = expected_diagnostic
         arguments ||= {}
-        message     = Parser::MESSAGES[reason] % arguments
+        message     = Parser::Messages.compile(reason, arguments)
 
         assert_equal level, actual_diagnostic.level
         assert_equal reason, actual_diagnostic.reason
@@ -227,8 +223,7 @@ module ParseHelper
 
   def refute_diagnoses(code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(refute_diagnoses)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(refute_diagnoses)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -244,16 +239,15 @@ module ParseHelper
 
   def assert_context(context, code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_context)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_context)', source: code)
 
-      begin
-        parser.parse(source_file)
-      rescue Parser::SyntaxError
-        # do nothing; the diagnostic was reported
-      end
+      parsed_ast = parser.parse(source_file)
 
-      assert_equal parser.context.stack, context, "(#{version}) parsing context"
+      nodes = find_matching_nodes(parsed_ast) { |node| node.type == :send && node.children[1] == :get_context }
+      assert_equal 1, nodes.count, "there must exactly 1 `get_context()` call"
+
+      node = nodes.first
+      assert_equal context, node.context, "(#{version}) expect parsing context to match"
     end
   end
 
@@ -316,4 +310,14 @@ module ParseHelper
       matching_children[index]
     end
   end
+
+  def find_matching_nodes(ast, &block)
+    return [] unless ast.is_a?(AST::Node)
+
+    result = []
+    result << ast if block.call(ast)
+    ast.children.each { |child| result += find_matching_nodes(child, &block) }
+
+    result
+  end
 end

data/test/test_ast_processor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'helper'
+
+class TestASTProcessor < Minitest::Test
+  LEAF_NODES = %i[
+    sym str int float complex rational
+    true false nil self
+    __FILE__ __LINE__ __ENCODING__
+    cbase regopt zsuper
+    match_with_trailing_comma match_nil_pattern
+    forward_args forwarded_args numargs kwnilarg
+    objc_varargs objc_restarg objc_kwarg
+    ident
+  ].freeze
+
+  def setup
+    @traversible = Parser::AST::Processor
+      .instance_methods(false)
+      .map { |mid| mid.to_s.scan(/\Aon_(.*)/) }
+      .flatten
+      .map(&:to_sym)
+
+    @traversible += LEAF_NODES
+  end
+
+  def test_nodes_are_traversible
+    for_each_node do |node|
+      assert_includes @traversible, node.type
+    end
+  end
+end

data/test/test_base.rb

@@ -26,6 +26,6 @@ class TestBase < Minitest::Test
   def test_loc_dup
     ast = Parser::CurrentRuby.parse('1')
     assert_nil ast.loc.dup.node
-    Parser::AST::Node.new(:root, [], :location => ast.loc)
+    Parser::AST::Node.new(:zsuper, [], :location => ast.loc)
   end
 end

data/test/test_current.rb

@@ -22,6 +22,8 @@ class TestCurrent < Minitest::Test
       assert_equal Parser::Ruby26, Parser::CurrentRuby
     when /^2\.7\.\d+/
       assert_equal Parser::Ruby27, Parser::CurrentRuby
+    when /^2\.8\.\d+/
+      assert_equal Parser::Ruby28, Parser::CurrentRuby
     else
       flunk "Update test_current for #{RUBY_VERSION}"
     end

data/test/test_diagnostic.rb

@@ -4,8 +4,8 @@ require 'helper'
 
 class TestDiagnostic < Minitest::Test
   def setup
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = 'if (this is some bad code + bugs)'
+    @buffer = Parser::Source::Buffer.new('(string)',
+      source: 'if (this is some bad code + bugs)')
 
     @range1 = Parser::Source::Range.new(@buffer, 0, 2) # if
     @range2 = Parser::Source::Range.new(@buffer, 4, 8) # this
@@ -16,7 +16,7 @@ class TestDiagnostic < Minitest::Test
       Parser::Diagnostic.new(:foobar, :escape_eof, {}, @range1)
     end
 
-    assert_match /level/, error.message
+    assert_match(/level/, error.message)
   end
 
   def test_freezes
@@ -50,8 +50,8 @@ class TestDiagnostic < Minitest::Test
   end
 
   def test_multiline_render
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = "abc abc abc\ndef def def\nghi ghi ghi\n"
+    @buffer = Parser::Source::Buffer.new('(string)',
+      source: "abc abc abc\ndef def def\nghi ghi ghi\n")
 
     location = Parser::Source::Range.new(@buffer, 4, 27)
 
@@ -80,8 +80,7 @@ class TestDiagnostic < Minitest::Test
   }
 }
     CODE
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = source
+    @buffer = Parser::Source::Buffer.new('(string)', source: source)
 
     location = Parser::Source::Range.new(@buffer, 33, 34)
     diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'tNL' },

data/test/test_diagnostic_engine.rb

@@ -4,9 +4,6 @@ require 'helper'
 
 class TestDiagnosticEngine < Minitest::Test
   def setup
-    @buffer  = Parser::Source::Buffer.new('(source)')
-    @buffer.source = 'foobar'
-
     @engine = Parser::Diagnostic::Engine.new
 
     @queue  = []
@@ -14,7 +11,7 @@ class TestDiagnosticEngine < Minitest::Test
   end
 
   def test_process_warnings
-    warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
+    warn = Parser::Diagnostic.new(:warning, :invalid_escape, {}, 1..2)
     @engine.process(warn)
 
     assert_equal [warn], @queue
@@ -23,7 +20,7 @@ class TestDiagnosticEngine < Minitest::Test
   def test_ignore_warnings
     @engine.ignore_warnings = true
 
-    warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
+    warn = Parser::Diagnostic.new(:warning, :invalid_escape, {}, 1..2)
     @engine.process(warn)
 
     assert_equal [], @queue
@@ -32,7 +29,7 @@ class TestDiagnosticEngine < Minitest::Test
   def test_all_errors_are_fatal
     @engine.all_errors_are_fatal = true
 
-    error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
+    error = Parser::Diagnostic.new(:error, :invalid_escape, {}, 1..2)
 
     err = assert_raises Parser::SyntaxError do
       @engine.process(error)
@@ -44,14 +41,14 @@ class TestDiagnosticEngine < Minitest::Test
   end
 
   def test_all_errors_are_collected
-    error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
+    error = Parser::Diagnostic.new(:error, :invalid_escape, {}, 1..2)
     @engine.process(error)
 
     assert_equal [error], @queue
   end
 
   def test_fatal_error
-    fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, @buffer, 1..2)
+    fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, {}, 1..2)
 
     assert_raises Parser::SyntaxError do
       @engine.process(fatal)

data/test/test_lexer.rb

@@ -37,9 +37,8 @@ class TestLexer < Minitest::Test
   end
 
   def assert_escape(expected, input)
-    source_buffer = Parser::Source::Buffer.new('(assert_escape)')
-
-    source_buffer.source = "\"\\#{input}\"".encode(input.encoding)
+    source_buffer = Parser::Source::Buffer.new('(assert_escape)',
+      source: "\"\\#{input}\"".encode(input.encoding))
 
     @lex.reset
     @lex.source_buffer = source_buffer
@@ -71,8 +70,7 @@ class TestLexer < Minitest::Test
   end
 
   def assert_scanned(input, *args)
-    source_buffer = Parser::Source::Buffer.new('(assert_scanned)')
-    source_buffer.source = input
+    source_buffer = Parser::Source::Buffer.new('(assert_scanned)', source: input)
 
     @lex.reset(false)
     @lex.source_buffer = source_buffer
@@ -3569,14 +3567,25 @@ class TestLexer < Minitest::Test
                    :tIDENTIFIER, 're', [1, 3])
   end
 
+  def test_endless_method
+    setup_lexer(28)
+
+    assert_scanned('def foo() = 42',
+                    :kDEF, "def", [0, 3],
+                    :tIDENTIFIER, "foo", [4, 7],
+                    :tLPAREN2, "(", [7, 8],
+                    :tRPAREN, ")", [8, 9],
+                    :tEQL, "=", [10, 11],
+                    :tINTEGER, 42, [12, 14])
+  end
+
   def lex_numbered_parameter(input)
     @lex.max_numparam_stack.push
 
     @lex.context = Parser::Context.new
     @lex.context.push(:block)
 
-    source_buffer = Parser::Source::Buffer.new('(assert_lex_numbered_parameter)')
-    source_buffer.source = input
+    source_buffer = Parser::Source::Buffer.new('(assert_lex_numbered_parameter)', source: input)
 
     @lex.source_buffer = source_buffer
 
@@ -3594,7 +3603,7 @@ class TestLexer < Minitest::Test
 
   def refute_scanned_numbered_parameter(input, message = nil)
     err = assert_raises Parser::SyntaxError do
-      lex_token, (lex_value, lex_range) = lex_numbered_parameter(input)
+      _lex_token, (_lex_value, _lex_range) = lex_numbered_parameter(input)
     end
 
     if message