parser 2.7.1.0 → 2.7.1.5

data/lib/parser/tree_rewriter.rb

@@ -28,8 +28,7 @@ module Parser
   #     EOF
   #
   #     ast           = Parser::CurrentRuby.parse code
-  #     buffer        = Parser::Source::Buffer.new('(example)')
-  #     buffer.source = code
+  #     buffer        = Parser::Source::Buffer.new('(example)', source: code)
   #     rewriter      = RemoveDo.new
   #
   #     # Rewrite the AST, returns a String with the new form.

data/lib/parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parser
-  VERSION = '2.7.1.0'
+  VERSION = '2.7.1.5'
 end

data/parser.gemspec

@@ -32,19 +32,23 @@ Gem::Specification.new do |spec|
                           lib/parser/ruby25.rb
                           lib/parser/ruby26.rb
                           lib/parser/ruby27.rb
+                          lib/parser/ruby30.rb
                           lib/parser/macruby.rb
                           lib/parser/rubymotion.rb
+                       ) - %w(
+                          .gitignore
                        )
+
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^test/})
   spec.require_paths = ['lib']
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_dependency             'ast',       '~> 2.4.0'
+  spec.add_dependency             'ast',       '~> 2.4.1'
 
   spec.add_development_dependency 'bundler',   '>= 1.15', '< 3.0.0'
-  spec.add_development_dependency 'rake',      '~> 10.0'
+  spec.add_development_dependency 'rake',      '~> 13.0.1'
   spec.add_development_dependency 'racc',      '= 1.4.15'
   spec.add_development_dependency 'cliver',    '~> 0.3.2'
 

data/test/helper.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require 'tempfile'
-require 'minitest/test'
-
 require 'simplecov'
 
 if ENV.include?('COVERAGE') && SimpleCov.usable?
@@ -20,6 +18,7 @@ if ENV.include?('COVERAGE') && SimpleCov.usable?
       ruby25.y
       ruby26.y
       ruby27.y
+      ruby30.y
     ),
     File.expand_path('../../lib/parser', __FILE__))
 
@@ -27,9 +26,9 @@ if ENV.include?('COVERAGE') && SimpleCov.usable?
   at_exit { RaccCoverage.stop }
 
   SimpleCov.start do
-    self.formatter = SimpleCov::Formatter::MultiFormatter[
-      SimpleCov::Formatter::HTMLFormatter,
-    ]
+    self.formatter = SimpleCov::Formatter::MultiFormatter.new(
+      SimpleCov::Formatter::HTMLFormatter
+    )
 
     add_group 'Grammars' do |source_file|
       source_file.filename =~ %r{\.y$}
@@ -52,9 +51,53 @@ require 'minitest/autorun'
 $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
 require 'parser'
 
+module NodeCollector
+  extend self
+  attr_accessor :callbacks, :nodes
+  self.callbacks = []
+  self.nodes = []
+
+  def check
+    @callbacks.each do |callback|
+      @nodes.each { |node| callback.call(node) }
+    end
+    puts "#{callbacks.size} additional tests on #{nodes.size} nodes ran successfully"
+  end
+
+  Minitest.after_run { check }
+end
+
+def for_each_node(&block)
+  NodeCollector.callbacks << block
+end
+
 class Parser::AST::Node
   def initialize(type, *)
-    raise "Type #{type} missing from Parser::Meta::NODE_TYPES" unless Parser::Meta::NODE_TYPES.include?(type)
+    NodeCollector.nodes << self
     super
   end
 end
+
+# Special test extension that records a context of the parser
+# for any node that is created
+module NodeContextExt
+  module NodeExt
+    attr_reader :context
+
+    def assign_properties(properties)
+      super
+
+      if (context = properties[:context])
+        @context = context
+      end
+    end
+  end
+  Parser::AST::Node.prepend(NodeExt)
+
+  module BuilderExt
+    def n(type, children, source_map)
+      super.updated(nil, nil, context: @parser.context.stack.dup)
+    end
+  end
+  Parser::Builders::Default.prepend(BuilderExt)
+end

data/test/parse_helper.rb

@@ -7,7 +7,7 @@ module ParseHelper
   require 'parser/macruby'
   require 'parser/rubymotion'
 
-  ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 mac ios)
+  ALL_VERSIONS = %w(1.8 1.9 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 mac ios)
 
   def setup
     @diagnostics = []
@@ -27,6 +27,7 @@ module ParseHelper
     when '2.5' then parser = Parser::Ruby25.new
     when '2.6' then parser = Parser::Ruby26.new
     when '2.7' then parser = Parser::Ruby27.new
+    when '3.0' then parser = Parser::Ruby30.new
     when 'mac' then parser = Parser::MacRuby.new
     when 'ios' then parser = Parser::RubyMotion.new
     else raise "Unrecognized Ruby version #{version}"
@@ -48,15 +49,17 @@ module ParseHelper
     end
   end
 
-  def assert_source_range(begin_pos, end_pos, range, version, what)
-    assert range.is_a?(Parser::Source::Range),
-           "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
-
-    assert_equal begin_pos, range.begin_pos,
-                 "(#{version}) begin of #{what}"
-
-    assert_equal end_pos, range.end_pos,
-                 "(#{version}) end of #{what}"
+  def assert_source_range(expect_range, range, version, what)
+    if expect_range == nil
+      # Avoid "Use assert_nil if expecting nil from .... This will fail in Minitest 6.""
+      assert_nil range,
+                   "(#{version}) range of #{what}"
+    else
+      assert range.is_a?(Parser::Source::Range),
+             "(#{version}) #{range.inspect}.is_a?(Source::Range) for #{what}"
+      assert_equal expect_range, range.to_range,
+                   "(#{version}) range of #{what}"
+    end
   end
 
   # Use like this:
@@ -84,8 +87,7 @@ module ParseHelper
   end
 
   def try_parsing(ast, code, parser, source_maps, version)
-    source_file = Parser::Source::Buffer.new('(assert_parses)')
-    source_file.source = code
+    source_file = Parser::Source::Buffer.new('(assert_parses)', source: code)
 
     begin
       parsed_ast = parser.parse(source_file)
@@ -105,8 +107,7 @@ module ParseHelper
     assert_equal ast, parsed_ast,
                  "(#{version}) AST equality"
 
-    parse_source_map_descriptions(source_maps) \
-        do |begin_pos, end_pos, map_field, ast_path, line|
+    parse_source_map_descriptions(source_maps) do |range, map_field, ast_path, line|
 
       astlet = traverse_ast(parsed_ast, ast_path)
 
@@ -120,9 +121,9 @@ module ParseHelper
       assert astlet.location.respond_to?(map_field),
              "(#{version}) #{astlet.location.inspect}.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
 
-      range = astlet.location.send(map_field)
+      found_range = astlet.location.send(map_field)
 
-      assert_source_range(begin_pos, end_pos, range, version, line.inspect)
+      assert_source_range(range, found_range, version, line.inspect)
     end
 
     assert parser.instance_eval { @lexer }.cmdarg.empty?,
@@ -142,8 +143,7 @@ module ParseHelper
   # ~~~
   def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_diagnoses)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -159,26 +159,25 @@ module ParseHelper
 
       level, reason, arguments = diagnostic
       arguments ||= {}
-      message     = Parser::MESSAGES[reason] % arguments
+      message     = Parser::Messages.compile(reason, arguments)
 
       assert_equal level, emitted_diagnostic.level
       assert_equal reason, emitted_diagnostic.reason
       assert_equal arguments, emitted_diagnostic.arguments
       assert_equal message, emitted_diagnostic.message
 
-      parse_source_map_descriptions(source_maps) \
-          do |begin_pos, end_pos, map_field, ast_path, line|
+      parse_source_map_descriptions(source_maps) do |range, map_field, ast_path, line|
 
         case map_field
         when 'location'
-          assert_source_range begin_pos, end_pos,
+          assert_source_range range,
                               emitted_diagnostic.location,
                               version, 'location'
 
         when 'highlights'
           index = ast_path.first.to_i
 
-          assert_source_range begin_pos, end_pos,
+          assert_source_range range,
                               emitted_diagnostic.highlights[index],
                               version, "#{index}th highlight"
 
@@ -201,8 +200,7 @@ module ParseHelper
   # ~~~
   def assert_diagnoses_many(diagnostics, code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_diagnoses_many)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -215,7 +213,7 @@ module ParseHelper
       diagnostics.zip(@diagnostics) do |expected_diagnostic, actual_diagnostic|
         level, reason, arguments = expected_diagnostic
         arguments ||= {}
-        message     = Parser::MESSAGES[reason] % arguments
+        message     = Parser::Messages.compile(reason, arguments)
 
         assert_equal level, actual_diagnostic.level
         assert_equal reason, actual_diagnostic.reason
@@ -227,8 +225,7 @@ module ParseHelper
 
   def refute_diagnoses(code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(refute_diagnoses)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(refute_diagnoses)', source: code)
 
       begin
         parser = parser.parse(source_file)
@@ -244,23 +241,22 @@ module ParseHelper
 
   def assert_context(context, code, versions=ALL_VERSIONS)
     with_versions(versions) do |version, parser|
-      source_file = Parser::Source::Buffer.new('(assert_context)')
-      source_file.source = code
+      source_file = Parser::Source::Buffer.new('(assert_context)', source: code)
 
-      begin
-        parser.parse(source_file)
-      rescue Parser::SyntaxError
-        # do nothing; the diagnostic was reported
-      end
+      parsed_ast = parser.parse(source_file)
+
+      nodes = find_matching_nodes(parsed_ast) { |node| node.type == :send && node.children[1] == :get_context }
+      assert_equal 1, nodes.count, "there must exactly 1 `get_context()` call"
 
-      assert_equal parser.context.stack, context, "(#{version}) parsing context"
+      node = nodes.first
+      assert_equal context, node.context, "(#{version}) expect parsing context to match"
     end
   end
 
   SOURCE_MAP_DESCRIPTION_RE =
       /(?x)
        ^(?# $1 skip)            ^(\s*)
-        (?# $2 highlight)        ([~\^]+)
+        (?# $2 highlight)        ([~\^]+|\!)
                                  \s+
         (?# $3 source_map_field) ([a-z_]+)
         (?# $5 ast_path)         (\s+\(([a-z_.\/0-9]+)\))?
@@ -278,8 +274,11 @@ module ParseHelper
       next if line.empty?
 
       if (match = SOURCE_MAP_DESCRIPTION_RE.match(line))
-        begin_pos        = match[1].length
-        end_pos          = begin_pos + match[2].length
+        if match[2] != '!'
+          begin_pos        = match[1].length
+          end_pos          = begin_pos + match[2].length
+          range            = begin_pos...end_pos
+        end
         source_map_field = match[3]
 
         if match[5]
@@ -288,7 +287,7 @@ module ParseHelper
           ast_path = []
         end
 
-        yield begin_pos, end_pos, source_map_field, ast_path, line
+        yield range, source_map_field, ast_path, line
       else
         raise "Cannot parse source map description line: #{line.inspect}."
       end
@@ -316,4 +315,14 @@ module ParseHelper
       matching_children[index]
     end
   end
+
+  def find_matching_nodes(ast, &block)
+    return [] unless ast.is_a?(AST::Node)
+
+    result = []
+    result << ast if block.call(ast)
+    ast.children.each { |child| result += find_matching_nodes(child, &block) }
+
+    result
+  end
 end

data/test/test_ast_processor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'helper'
+
+class TestASTProcessor < Minitest::Test
+  LEAF_NODES = %i[
+    sym str int float complex rational
+    true false nil self
+    __FILE__ __LINE__ __ENCODING__
+    cbase regopt zsuper
+    match_with_trailing_comma match_nil_pattern
+    forward_args forwarded_args numargs kwnilarg
+    objc_varargs objc_restarg objc_kwarg
+    ident
+  ].freeze
+
+  def setup
+    @traversible = Parser::AST::Processor
+      .instance_methods(false)
+      .map { |mid| mid.to_s.scan(/\Aon_(.*)/) }
+      .flatten
+      .map(&:to_sym)
+
+    @traversible += LEAF_NODES
+  end
+
+  def test_nodes_are_traversible
+    for_each_node do |node|
+      assert_includes @traversible, node.type
+    end
+  end
+end

data/test/test_base.rb

@@ -26,6 +26,6 @@ class TestBase < Minitest::Test
   def test_loc_dup
     ast = Parser::CurrentRuby.parse('1')
     assert_nil ast.loc.dup.node
-    Parser::AST::Node.new(:root, [], :location => ast.loc)
+    Parser::AST::Node.new(:zsuper, [], :location => ast.loc)
   end
 end

data/test/test_current.rb

@@ -22,6 +22,8 @@ class TestCurrent < Minitest::Test
       assert_equal Parser::Ruby26, Parser::CurrentRuby
     when /^2\.7\.\d+/
       assert_equal Parser::Ruby27, Parser::CurrentRuby
+    when /^3\.0\.\d+/
+      assert_equal Parser::Ruby30, Parser::CurrentRuby
     else
       flunk "Update test_current for #{RUBY_VERSION}"
     end

data/test/test_diagnostic.rb

@@ -4,8 +4,8 @@ require 'helper'
 
 class TestDiagnostic < Minitest::Test
   def setup
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = 'if (this is some bad code + bugs)'
+    @buffer = Parser::Source::Buffer.new('(string)',
+      source: 'if (this is some bad code + bugs)')
 
     @range1 = Parser::Source::Range.new(@buffer, 0, 2) # if
     @range2 = Parser::Source::Range.new(@buffer, 4, 8) # this
@@ -16,7 +16,7 @@ class TestDiagnostic < Minitest::Test
       Parser::Diagnostic.new(:foobar, :escape_eof, {}, @range1)
     end
 
-    assert_match /level/, error.message
+    assert_match(/level/, error.message)
   end
 
   def test_freezes
@@ -50,8 +50,8 @@ class TestDiagnostic < Minitest::Test
   end
 
   def test_multiline_render
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = "abc abc abc\ndef def def\nghi ghi ghi\n"
+    @buffer = Parser::Source::Buffer.new('(string)',
+      source: "abc abc abc\ndef def def\nghi ghi ghi\n")
 
     location = Parser::Source::Range.new(@buffer, 4, 27)
 
@@ -80,8 +80,7 @@ class TestDiagnostic < Minitest::Test
   }
 }
     CODE
-    @buffer = Parser::Source::Buffer.new('(string)')
-    @buffer.source = source
+    @buffer = Parser::Source::Buffer.new('(string)', source: source)
 
     location = Parser::Source::Range.new(@buffer, 33, 34)
     diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'tNL' },

data/test/test_diagnostic_engine.rb

@@ -4,9 +4,6 @@ require 'helper'
 
 class TestDiagnosticEngine < Minitest::Test
   def setup
-    @buffer  = Parser::Source::Buffer.new('(source)')
-    @buffer.source = 'foobar'
-
     @engine = Parser::Diagnostic::Engine.new
 
     @queue  = []
@@ -14,7 +11,7 @@ class TestDiagnosticEngine < Minitest::Test
   end
 
   def test_process_warnings
-    warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
+    warn = Parser::Diagnostic.new(:warning, :invalid_escape, {}, 1..2)
     @engine.process(warn)
 
     assert_equal [warn], @queue
@@ -23,7 +20,7 @@ class TestDiagnosticEngine < Minitest::Test
   def test_ignore_warnings
     @engine.ignore_warnings = true
 
-    warn = Parser::Diagnostic.new(:warning, :invalid_escape, @buffer, 1..2)
+    warn = Parser::Diagnostic.new(:warning, :invalid_escape, {}, 1..2)
     @engine.process(warn)
 
     assert_equal [], @queue
@@ -32,7 +29,7 @@ class TestDiagnosticEngine < Minitest::Test
   def test_all_errors_are_fatal
     @engine.all_errors_are_fatal = true
 
-    error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
+    error = Parser::Diagnostic.new(:error, :invalid_escape, {}, 1..2)
 
     err = assert_raises Parser::SyntaxError do
       @engine.process(error)
@@ -44,14 +41,14 @@ class TestDiagnosticEngine < Minitest::Test
   end
 
   def test_all_errors_are_collected
-    error = Parser::Diagnostic.new(:error, :invalid_escape, @buffer, 1..2)
+    error = Parser::Diagnostic.new(:error, :invalid_escape, {}, 1..2)
     @engine.process(error)
 
     assert_equal [error], @queue
   end
 
   def test_fatal_error
-    fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, @buffer, 1..2)
+    fatal = Parser::Diagnostic.new(:fatal, :invalid_escape, {}, 1..2)
 
     assert_raises Parser::SyntaxError do
       @engine.process(fatal)