parser 2.7.1.3 → 2.7.1.4

data/lib/parser/rubymotion.y

@@ -1018,11 +1018,15 @@ rule
                       result      = @builder.block(val[0],
                                       begin_t, args, body, end_t)
                     }
-                | tLAMBDA lambda
+                | tLAMBDA
+                    {
+                      @context.push(:lambda)
+                    }
+                  lambda
                     {
                       lambda_call = @builder.call_lambda(val[0])
 
-                      args, (begin_t, body, end_t) = val[1]
+                      args, (begin_t, body, end_t) = val[2]
                       result      = @builder.block(lambda_call,
                                       begin_t, args, body, end_t)
                     }
@@ -1136,6 +1140,7 @@ rule
                     {
                       @static_env.extend_static
                       @lexer.push_cmdarg
+                      @context.push(:module)
                     }
                     bodystmt kEND
                     {
@@ -1148,6 +1153,7 @@ rule
 
                       @lexer.pop_cmdarg
                       @static_env.unextend
+                      @context.pop
                     }
                 | kDEF fname
                     {
@@ -1429,9 +1435,13 @@ rule
           lambda:   {
                       @static_env.extend_dynamic
                     }
-                  f_larglist lambda_body
+                  f_larglist
+                    {
+                      @context.pop
+                    }
+                  lambda_body
                     {
-                      result = [ val[1], val[2] ]
+                      result = [ val[1], val[3] ]
 
                       @static_env.unextend
                     }

data/lib/parser/runner.rb

@@ -37,7 +37,7 @@ module Parser
 
     private
 
-    LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0].freeze
+    LEGACY_MODES = %i[lambda procarg0 encoding index arg_inside_procarg0 forward_arg].freeze
 
     def runner_name
       raise NotImplementedError, "implement #{self.class}##{__callee__}"

data/lib/parser/source/range.rb

@@ -13,7 +13,7 @@ module Parser
     #        ^^
     #
     # @!attribute [r] source_buffer
-    #  @return [Parser::Diagnostic::Engine]
+    #  @return [Parser::Source::Buffer]
     #
     # @!attribute [r] begin_pos
     #  @return [Integer] index of the first character in the range

data/lib/parser/source/tree_rewriter.rb

@@ -129,6 +129,8 @@ module Parser
       ##
       # Merges the updates of argument with the receiver.
       # Policies of the receiver are used.
+      # This action is atomic in that it won't change the receiver
+      # unless it succeeds.
       #
       # @param [Rewriter] with
       # @return [Rewriter] self
@@ -154,6 +156,32 @@ module Parser
         dup.merge!(with)
       end
 
+      ##
+      # For special cases where one needs to merge a rewriter attached to a different source_buffer
+      # or that needs to be offset. Policies of the receiver are used.
+      #
+      # @param [TreeRewriter] rewriter from different source_buffer
+      # @param [Integer] offset
+      # @return [Rewriter] self
+      # @raise [IndexError] if action ranges (once offset) don't fit the current buffer
+      #
+      def import!(foreign_rewriter, offset: 0)
+        return self if foreign_rewriter.empty?
+
+        contracted = foreign_rewriter.action_root.contract
+        merge_effective_range = ::Parser::Source::Range.new(
+          @source_buffer,
+          contracted.range.begin_pos + offset,
+          contracted.range.end_pos + offset,
+        )
+        check_range_validity(merge_effective_range)
+
+        merge_with = contracted.moved(@source_buffer, offset)
+
+        @action_root = @action_root.combine(merge_with)
+        self
+      end
+
       ##
       # Replaces the code of the source range `range` with `content`.
       #
@@ -234,6 +262,44 @@ module Parser
         chunks.join
       end
 
+      ##
+      # Returns a representation of the rewriter as an ordered list of replacements.
+      #
+      #     rewriter.as_replacements # => [ [1...1, '('],
+      #                                     [2...4, 'foo'],
+      #                                     [5...6, ''],
+      #                                     [6...6, '!'],
+      #                                     [10...10, ')'],
+      #                                   ]
+      #
+      # This representation is sufficient to recreate the result of `process` but it is
+      # not sufficient to recreate completely the rewriter for further merging/actions.
+      # See `as_nested_actions`
+      #
+      # @return [Array<Range, String>] an ordered list of pairs of range & replacement
+      #
+      def as_replacements
+        @action_root.ordered_replacements
+      end
+
+      ##
+      # Returns a representation of the rewriter as nested insertions (:wrap) and replacements.
+      #
+      #     rewriter.as_actions # =>[ [:wrap, 1...10, '(', ')'],
+      #                               [:wrap, 2...6, '', '!'],  # aka "insert_after"
+      #                               [:replace, 2...4, 'foo'],
+      #                               [:replace, 5...6, ''],  # aka "removal"
+      #                             ],
+      #
+      # Contrary to `as_replacements`, this representation is sufficient to recreate exactly
+      # the rewriter.
+      #
+      # @return [Array<(Symbol, Range, String{, String})>]
+      #
+      def as_nested_actions
+        @action_root.nested_actions
+      end
+
       ##
       # Provides a protected block where a sequence of multiple rewrite actions
       # are handled atomically. If any of the actions failed by clobbering,
@@ -310,7 +376,7 @@ module Parser
 
       def check_range_validity(range)
         if range.begin_pos < 0 || range.end_pos > @source_buffer.source.size
-          raise IndexError, "The range #{range} is outside the bounds of the source"
+          raise IndexError, "The range #{range.to_range} is outside the bounds of the source"
         end
         range
       end

data/lib/parser/source/tree_rewriter/action.rb

@@ -46,10 +46,49 @@ module Parser
         reps
       end
 
+      def nested_actions
+        actions = []
+        actions << [:wrap, @range, @insert_before, @insert_after] if !@insert_before.empty? ||
+                                                                     !@insert_after.empty?
+        actions << [:replace, @range, @replacement] if @replacement
+        actions.concat(@children.flat_map(&:nested_actions))
+      end
+
       def insertion?
         !insert_before.empty? || !insert_after.empty? || (replacement && !replacement.empty?)
       end
 
+      ##
+      # A root action has its range set to the whole source range, even
+      # though it typically do not act on that range.
+      # This method returns the action as if it was a child action with
+      # its range contracted.
+      # @return [Action]
+      def contract
+        raise 'Empty actions can not be contracted' if empty?
+        return self if insertion?
+        range = @range.with(
+          begin_pos: children.first.range.begin_pos,
+          end_pos: children.last.range.end_pos,
+        )
+        with(range: range)
+      end
+
+      ##
+      # @return [Action] that has been moved to the given source_buffer and with the given offset
+      # No check is done on validity of resulting range.
+      def moved(source_buffer, offset)
+        moved_range = ::Parser::Source::Range.new(
+          source_buffer,
+          @range.begin_pos + offset,
+          @range.end_pos + offset
+        )
+        with(
+          range: moved_range,
+          children: children.map { |child| child.moved(source_buffer, offset) }
+        )
+      end
+
       protected
 
       attr_reader :children

data/lib/parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parser
-  VERSION = '2.7.1.3'
+  VERSION = '2.7.1.4'
 end

data/parser.gemspec

@@ -42,7 +42,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_dependency             'ast',       '~> 2.4.0'
+  spec.add_dependency             'ast',       '~> 2.4.1'
 
   spec.add_development_dependency 'bundler',   '>= 1.15', '< 3.0.0'
   spec.add_development_dependency 'rake',      '~> 13.0.1'

data/test/helper.rb

@@ -77,3 +77,27 @@ class Parser::AST::Node
     super
   end
 end
+
+# Special test extension that records a context of the parser
+# for any node that is created
+module NodeContextExt
+  module NodeExt
+    attr_reader :context
+
+    def assign_properties(properties)
+      super
+
+      if (context = properties[:context])
+        @context = context
+      end
+    end
+  end
+  Parser::AST::Node.prepend(NodeExt)
+
+  module BuilderExt
+    def n(type, children, source_map)
+      super.updated(nil, nil, context: @parser.context.stack.dup)
+    end
+  end
+  Parser::Builders::Default.prepend(BuilderExt)
+end

data/test/parse_helper.rb

@@ -241,13 +241,13 @@ module ParseHelper
     with_versions(versions) do |version, parser|
       source_file = Parser::Source::Buffer.new('(assert_context)', source: code)
 
-      begin
-        parser.parse(source_file)
-      rescue Parser::SyntaxError
-        # do nothing; the diagnostic was reported
-      end
+      parsed_ast = parser.parse(source_file)
 
-      assert_equal parser.context.stack, context, "(#{version}) parsing context"
+      nodes = find_matching_nodes(parsed_ast) { |node| node.type == :send && node.children[1] == :get_context }
+      assert_equal 1, nodes.count, "there must exactly 1 `get_context()` call"
+
+      node = nodes.first
+      assert_equal context, node.context, "(#{version}) expect parsing context to match"
     end
   end
 
@@ -310,4 +310,14 @@ module ParseHelper
       matching_children[index]
     end
   end
+
+  def find_matching_nodes(ast, &block)
+    return [] unless ast.is_a?(AST::Node)
+
+    result = []
+    result << ast if block.call(ast)
+    ast.children.each { |child| result += find_matching_nodes(child, &block) }
+
+    result
+  end
 end

data/test/test_parser.rb

@@ -6551,39 +6551,64 @@ class TestParser < Minitest::Test
 
   def test_context_class
     [
-      %q{class A;},
-      %q{class A < B;}
+      %q{class A; get_context; end},
+      %q{class A < B; get_context; end}
     ].each do |code|
       assert_context([:class], code, ALL_VERSIONS)
     end
   end
 
+  def test_context_module
+    assert_context(
+      [:module],
+      %q{module M; get_context; end},
+      ALL_VERSIONS)
+  end
+
   def test_context_sclass
     assert_context(
       [:sclass],
-      %q{class << foo;},
+      %q{class << foo; get_context; end},
       ALL_VERSIONS)
   end
 
   def test_context_def
-    assert_context(
-      [:def],
-      %q{def m;},
-      ALL_VERSIONS)
+    [
+      %q{def m; get_context; end},
+      %q{def m(a = get_context); end}
+    ].each do |code|
+      assert_context([:def], code, ALL_VERSIONS)
+    end
+
+    [
+      %q{def m() = get_context},
+      %q{def m(a = get_context) = 42}
+    ].each do |code|
+      assert_context([:def], code, SINCE_2_8)
+    end
   end
 
   def test_context_defs
-    assert_context(
-      [:defs],
-      %q{def foo.m;},
-      ALL_VERSIONS)
+    [
+      %q{def foo.m; get_context; end},
+      %q{def foo.m(a = get_context); end}
+    ].each do |code|
+      assert_context([:defs], code, ALL_VERSIONS)
+    end
+
+    [
+      %q{def foo.m() = get_context},
+      %q{def foo.m(a = get_context) = 42}
+    ].each do |code|
+      assert_context([:defs], code, SINCE_2_8)
+    end
   end
 
   def test_context_cmd_brace_block
     [
-      'tap foo {',
-      'foo.tap foo {',
-      'foo::tap foo {'
+      'tap foo { get_context }',
+      'foo.tap foo { get_context }',
+      'foo::tap foo { get_context }'
     ].each do |code|
       assert_context([:block], code, ALL_VERSIONS)
     end
@@ -6591,12 +6616,12 @@ class TestParser < Minitest::Test
 
   def test_context_brace_block
     [
-      'tap {',
-      'foo.tap {',
-      'foo::tap {',
-      'tap do',
-      'foo.tap do',
-      'foo::tap do'
+      'tap { get_context }',
+      'foo.tap { get_context }',
+      'foo::tap { get_context }',
+      'tap do get_context end',
+      'foo.tap do get_context end',
+      'foo::tap do get_context end'
     ].each do |code|
       assert_context([:block], code, ALL_VERSIONS)
     end
@@ -6604,9 +6629,9 @@ class TestParser < Minitest::Test
 
   def test_context_do_block
     [
-      %q{tap 1 do},
-      %q{foo.tap do},
-      %q{foo::tap do}
+      %q{tap 1 do get_context end},
+      %q{foo.tap do get_context end},
+      %q{foo::tap do get_context end}
     ].each do |code|
       assert_context([:block], code, ALL_VERSIONS)
     end
@@ -6614,8 +6639,12 @@ class TestParser < Minitest::Test
 
   def test_context_lambda
     [
-      '->() {',
-      '->() do'
+      '->() { get_context }',
+      '->() do get_context end',
+      '-> { get_context }',
+      '-> do get_context end',
+      '->(a = get_context) {}',
+      '->(a = get_context) do end'
     ].each do |code|
       assert_context([:lambda], code, SINCE_1_9)
     end
@@ -6623,28 +6652,61 @@ class TestParser < Minitest::Test
 
   def test_context_nested
     assert_context(
-      [:class, :sclass, :defs, :def, :block],
-      %q{class A; class << foo; def bar.m; def m; tap do},
+      [:class, :module, :sclass, :defs, :def, :block],
+      %q{
+        class A
+          module M
+            class << foo
+              def bar.m
+                def m
+                  tap do
+                    get_context
+                  end
+                end
+              end
+            end
+          end
+        end
+      },
       ALL_VERSIONS)
 
     assert_context(
-      [:class, :sclass, :defs, :def, :lambda, :block],
-      %q{class A; class << foo; def bar.m; def m; -> do; tap do},
+      [:class, :module, :sclass, :defs, :def, :lambda, :block],
+      %q{
+        class A
+          module M
+            class << foo
+              def bar.m
+                def m
+                  -> do
+                    tap do
+                      get_context
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      },
       SINCE_1_9)
 
     assert_context(
       [],
       %q{
         class A
-          class << foo
-            def bar.m
-              def m
-                tap do
+          module M
+            class << foo
+              def bar.m
+                def m
+                  tap do
+                  end
                 end
               end
             end
           end
         end
+        get_context
       },
       ALL_VERSIONS)
 
@@ -6652,17 +6714,20 @@ class TestParser < Minitest::Test
       [],
       %q{
         class A
-          class << foo
-            def bar.m
-              def m
-                -> do
-                  tap do
+          module M
+            class << foo
+              def bar.m
+                def m
+                  -> do
+                    tap do
+                    end
                   end
                 end
               end
             end
           end
         end
+        get_context
       },
       SINCE_1_9)
   end
@@ -7746,7 +7811,8 @@ class TestParser < Minitest::Test
     end
   end
 
-  def test_forward_args
+  def test_forward_args_legacy
+    Parser::Builders::Default.emit_forward_arg = false
     assert_parses(
       s(:def, :foo,
         s(:forward_args),
@@ -7778,7 +7844,27 @@ class TestParser < Minitest::Test
       %q{def foo(...); end},
       %q{},
       SINCE_2_7)
+  ensure
+    Parser::Builders::Default.emit_forward_arg = true
+  end
+
+  def test_forward_arg
+    assert_parses(
+      s(:def, :foo,
+        s(:args,
+          s(:forward_arg)),
+        s(:send, nil, :bar,
+          s(:forwarded_args))),
+      %q{def foo(...); bar(...); end},
+      %q{       ~ begin (args)
+        |       ~~~~~ expression (args)
+        |           ~ end (args)
+        |        ~~~ expression (args.forward_arg)
+        |                  ~~~ expression (send.forwarded_args)},
+      SINCE_2_7)
+  end
 
+  def test_forward_args_invalid
     assert_diagnoses(
       [:error, :block_and_blockarg],
       %q{def foo(...) bar(...) { }; end},
@@ -7804,6 +7890,12 @@ class TestParser < Minitest::Test
       %q{                    ^^^ location},
       SINCE_2_7)
 
+    assert_diagnoses(
+      [:error, :unexpected_token, { :token => 'tBDOT3' }],
+      %q{def foo(x,y,z); bar(x, y, z, ...); end},
+      %q{                             ^^^ location},
+      SINCE_2_8)
+
     assert_diagnoses(
       [:error, :unexpected_token, { :token => 'tBDOT3' }],
       %q{def foo(x,y,z); super(...); end},
@@ -7820,7 +7912,13 @@ class TestParser < Minitest::Test
       [:error, :unexpected_token, { :token => 'tBDOT3' }],
       %q{->(...) {}},
       %q{   ^^^ location},
-      SINCE_2_7)
+      ['2.7'])
+
+    assert_diagnoses(
+      [:error, :unexpected_token, { :token => 'tDOT3' }],
+      %q{->(...) {}},
+      %q{   ^^^ location},
+      SINCE_2_8)
 
     # Here and below the parser asssumes that
     # it can be a beginningless range, so the error comes after reducing right paren
@@ -7873,6 +7971,26 @@ class TestParser < Minitest::Test
       SINCE_2_7)
   end
 
+  def test_trailing_forward_arg
+    assert_parses(
+      s(:def, :foo,
+        s(:args,
+          s(:arg, :a),
+          s(:arg, :b),
+          s(:forward_arg)),
+        s(:send, nil, :bar,
+          s(:lvar, :a),
+          s(:int, 42),
+          s(:forwarded_args))),
+      %q{def foo(a, b, ...); bar(a, 42, ...); end},
+      %q{       ~ begin (args)
+        |       ~~~~~~~~~~~ expression (args)
+        |                 ~ end (args)
+        |              ~~~ expression (args.forward_arg)},
+      SINCE_2_8)
+  end
+
+
   def test_erange_without_parentheses_at_eol
     assert_diagnoses(
       [:warning, :triple_dot_at_eol],
@@ -8342,7 +8460,7 @@ class TestParser < Minitest::Test
         nil),
       "#{case_pre}#{code}; end",
       source_maps,
-      SINCE_2_7
+      versions
     )
   end
 
@@ -9491,7 +9609,10 @@ class TestParser < Minitest::Test
         |               ^ assignment
         |~~~~~~~~~~~~~~~~~~~~~~ expression},
       SINCE_2_8)
+  end
 
+  def test_endless_method_forwarded_args_legacy
+    Parser::Builders::Default.emit_forward_arg = false
     assert_parses(
       s(:def_e, :foo,
         s(:forward_args),
@@ -9503,6 +9624,7 @@ class TestParser < Minitest::Test
         |             ^ assignment
         |~~~~~~~~~~~~~~~~~~~~~~~ expression},
       SINCE_2_8)
+    Parser::Builders::Default.emit_forward_arg = true
   end
 
   def test_endless_method_without_brackets
@@ -9593,4 +9715,66 @@ class TestParser < Minitest::Test
       %{             ^^ location},
       SINCE_2_8)
   end
+
+  def test_find_pattern
+    assert_parses_pattern_match(
+      s(:in_pattern,
+        s(:find_pattern,
+          s(:match_rest,
+            s(:match_var, :x)),
+          s(:match_as,
+            s(:int, 1),
+            s(:match_var, :a)),
+          s(:match_rest,
+            s(:match_var, :y))),
+        nil,
+        s(:true)),
+      %q{in [*x, 1 => a, *y] then true},
+      %q{   ~~~~~~~~~~~~~~~~ expression (in_pattern.find_pattern)
+        |   ~ begin (in_pattern.find_pattern)
+        |                  ~ end (in_pattern.find_pattern)
+        |    ~~ expression (in_pattern.find_pattern.match_rest/1)
+        |                ~~ expression (in_pattern.find_pattern.match_rest/2)},
+      SINCE_2_8)
+
+    assert_parses_pattern_match(
+      s(:in_pattern,
+        s(:const_pattern,
+          s(:const, nil, :String),
+          s(:find_pattern,
+            s(:match_rest),
+            s(:int, 1),
+            s(:match_rest))),
+        nil,
+        s(:true)),
+      %q{in String(*, 1, *) then true},
+      %q{          ~~~~~~~ expression (in_pattern.const_pattern.find_pattern)},
+      SINCE_2_8)
+
+    assert_parses_pattern_match(
+      s(:in_pattern,
+        s(:const_pattern,
+          s(:const, nil, :Array),
+          s(:find_pattern,
+            s(:match_rest),
+            s(:int, 1),
+            s(:match_rest))),
+        nil,
+        s(:true)),
+      %q{in Array[*, 1, *] then true},
+      %q{         ~~~~~~~ expression (in_pattern.const_pattern.find_pattern)},
+      SINCE_2_8)
+
+    assert_parses_pattern_match(
+      s(:in_pattern,
+        s(:find_pattern,
+          s(:match_rest),
+          s(:int, 42),
+          s(:match_rest)),
+        nil,
+        s(:true)),
+      %q{in *, 42, * then true},
+      %q{   ~~~~~~~~ expression (in_pattern.find_pattern)},
+      SINCE_2_8)
+  end
 end