parser 2.7.1.3 → 3.0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bab855c4abc633b9ba13f855d816f20948ee539f27bea7878b574f4d0b3232f7
-  data.tar.gz: 929aa201a2c06738c6202c8efb2afd2dda77e630805738007c39f3b8274fa717
+  metadata.gz: dced4877039fb67469d83e8d10c513c2ca05cc09878b487e60a29a9795bef7f1
+  data.tar.gz: c7827038c9b8acfe02c7878e3f74c46184106ff1939a8fa8f0eb2d60c7560cc4
 SHA512:
-  metadata.gz: 78e2e37d1ef61d2c125c1462ecb2e48ab8f76a06fd97bcba28ea4acd5fa60f77a296f859fd6ad67a787027b7a9e62d0fc34bedc8c23522bce21213d1ca49d780
-  data.tar.gz: 07fa5ad8fe63b8503ced8fa0143062385f1d2f99e542f80cc32375269a9ad75f07cc2b0e00838232a71a9e797ba5cab28b661d8a58765f558e29353462641f06
+  metadata.gz: 6c1bc36f5b08e5ccb99d6dc7774a7e7041709b14d0ddecaa2aa63cacabd9e1be39ce28fbceb9c9c90512cd79d19e20d163386621694a9265fcdf6085c82aaf5e
+  data.tar.gz: 4c3f661941751ea6f66b2e1089fd5d53125e12b0c4819d7e80123143d1798736ba5169cc09b4f3be1dff50c56d8a609355c751c20b1236a3d7288c6d9fee9875

data/lib/parser.rb

@@ -46,7 +46,7 @@ module Parser
     require 'parser/source/map/variable'
     require 'parser/source/map/keyword'
     require 'parser/source/map/definition'
-    require 'parser/source/map/endless_definition'
+    require 'parser/source/map/method_definition'
     require 'parser/source/map/send'
     require 'parser/source/map/index'
     require 'parser/source/map/condition'

data/lib/parser/all.rb

@@ -10,4 +10,4 @@ require 'parser/ruby24'
 require 'parser/ruby25'
 require 'parser/ruby26'
 require 'parser/ruby27'
-require 'parser/ruby28'
+require 'parser/ruby30'

data/lib/parser/ast/processor.rb

@@ -20,6 +20,7 @@ module Parser
       alias on_array    process_regular_node
       alias on_pair     process_regular_node
       alias on_hash     process_regular_node
+      alias on_kwargs   process_regular_node
       alias on_irange   process_regular_node
       alias on_erange   process_regular_node
 
@@ -75,9 +76,6 @@ module Parser
       alias on_mlhs     process_regular_node
       alias on_masgn    process_regular_node
 
-      alias on_rasgn    process_regular_node
-      alias on_mrasgn   process_regular_node
-
       def on_const(node)
         scope_node, name = *node
 
@@ -127,6 +125,7 @@ module Parser
       alias on_kwarg          process_argument_node
       alias on_kwoptarg       process_argument_node
       alias on_kwrestarg      process_argument_node
+      alias on_forward_arg    process_argument_node
 
       def on_procarg0(node)
         if node.children[0].is_a?(Symbol)
@@ -162,8 +161,6 @@ module Parser
         ])
       end
 
-      alias on_def_e on_def
-
       def on_defs(node)
         definee_node, name, args_node, body_node = *node
 
@@ -173,8 +170,6 @@ module Parser
         ])
       end
 
-      alias on_defs_e on_defs
-
       alias on_undef    process_regular_node
       alias on_alias    process_regular_node
 
@@ -245,6 +240,8 @@ module Parser
 
       alias on_case_match              process_regular_node
       alias on_in_match                process_regular_node
+      alias on_match_pattern           process_regular_node
+      alias on_match_pattern_p         process_regular_node
       alias on_in_pattern              process_regular_node
       alias on_if_guard                process_regular_node
       alias on_unless_guard            process_regular_node
@@ -257,6 +254,7 @@ module Parser
       alias on_array_pattern_with_tail process_regular_node
       alias on_hash_pattern            process_regular_node
       alias on_const_pattern           process_regular_node
+      alias on_find_pattern            process_regular_node
 
       # @private
       def process_variable_node(node)

data/lib/parser/base.rb

@@ -109,6 +109,7 @@ module Parser
     end
     private_class_method :setup_source_buffer
 
+    attr_reader :lexer
     attr_reader :diagnostics
     attr_reader :builder
     attr_reader :static_env
@@ -177,16 +178,16 @@ module Parser
     end
 
     ##
-    # Parses a source buffer and returns the AST.
+    # Parses a source buffer and returns the AST, or `nil` in case of a non fatal error.
     #
     # @param [Parser::Source::Buffer] source_buffer The source buffer to parse.
-    # @return [Parser::AST::Node]
+    # @return [Parser::AST::Node, nil]
     #
     def parse(source_buffer)
       @lexer.source_buffer = source_buffer
       @source_buffer       = source_buffer
 
-      do_parse
+      do_parse || nil # Force `false` to `nil`, see https://github.com/ruby/racc/pull/136
     ensure
       # Don't keep references to the source file.
       @source_buffer       = nil
@@ -210,8 +211,9 @@ module Parser
 
     ##
     # Parses a source buffer and returns the AST, the source code comments,
-    # and the tokens emitted by the lexer. If `recover` is true and a fatal
-    # {SyntaxError} is encountered, `nil` is returned instead of the AST, and
+    # and the tokens emitted by the lexer. In case of a fatal error, a {SyntaxError}
+    # is raised, unless `recover` is true. In case of an error
+    # (non-fatal or recovered), `nil` is returned instead of the AST, and
     # comments as well as tokens are only returned up to the location of
     # the error.
     #

data/lib/parser/builders/default.rb

@@ -80,6 +80,8 @@ module Parser
       attr_accessor :emit_index
     end
 
+    @emit_index = false
+
     class << self
       ##
       # AST compatibility attribute; causes a single non-mlhs
@@ -95,7 +97,113 @@ module Parser
       attr_accessor :emit_arg_inside_procarg0
     end
 
-    @emit_index = false
+    @emit_arg_inside_procarg0 = false
+
+    class << self
+      ##
+      # AST compatibility attribute; arguments forwarding initially
+      # didn't have support for leading arguments
+      # (i.e. `def m(a, ...); end` was a syntax error). However, Ruby 3.0
+      # added support for any number of arguments in front of the `...`.
+      #
+      # If set to false (the default):
+      #   1. `def m(...) end` is emitted as
+      #      s(:def, :m, s(:forward_args), nil)
+      #   2. `def m(a, b, ...) end` is emitted as
+      #      s(:def, :m,
+      #        s(:args, s(:arg, :a), s(:arg, :b), s(:forward_arg)))
+      #
+      # If set to true it uses a single format:
+      #   1. `def m(...) end` is emitted as
+      #      s(:def, :m, s(:args, s(:forward_arg)))
+      #   2. `def m(a, b, ...) end` is emitted as
+      #      s(:def, :m, s(:args, s(:arg, :a), s(:arg, :b), s(:forward_arg)))
+      #
+      # It does't matter that much on 2.7 (because there can't be any leading arguments),
+      # but on 3.0 it should be better enabled to use a single AST format.
+      #
+      # @return [Boolean]
+      attr_accessor :emit_forward_arg
+    end
+
+    @emit_forward_arg = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from Ruby 2.7 keyword arguments
+      # of method calls that are passed explicitly as a hash (i.e. with curly braces)
+      # are treated as positional arguments and Ruby 2.7 emits a warning on such method
+      # call. Ruby 3.0 given an ArgumentError.
+      #
+      # If set to false (the default) the last hash argument is emitted as `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # If set to true it is emitted as `kwargs`:
+      #
+      # ```
+      # (send nil :foo
+      #   (kwargs
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # Note that `kwargs` node is just a replacement for `hash` argument,
+      # so if there's are multiple arguments (or a `kwsplat`) all of them
+      # are wrapped into `kwargs` instead of `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :a)
+      #       (int 42))
+      #     (kwsplat
+      #       (send nil :b))
+      #     (pair
+      #       (sym :c)
+      #       (int 10))))
+      # ```
+      attr_accessor :emit_kwargs
+    end
+
+    @emit_kwargs = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from 3.0 Ruby returns
+      # true/false from single-line pattern matching with `in` keyword.
+      #
+      # Before 3.0 there was an exception if given value doesn't match pattern.
+      #
+      # NOTE: This attribute affects only Ruby 2.7 grammar.
+      # 3.0 grammar always emits `match_pattern`/`match_pattern_p`
+      #
+      # If compatibility attribute set to false `foo in bar` is emitted as `in_match`:
+      #
+      # ```
+      # (in-match
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      #
+      # If set to true it's emitted as `match_pattern_p`:
+      # ```
+      # (match-pattern-p
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      attr_accessor :emit_match_pattern
+    end
+
+    @emit_match_pattern = false
 
     class << self
       ##
@@ -106,6 +214,9 @@ module Parser
         @emit_encoding = true
         @emit_index = true
         @emit_arg_inside_procarg0 = true
+        @emit_forward_arg = true
+        @emit_kwargs = true
+        @emit_match_pattern = true
       end
     end
 
@@ -278,18 +389,23 @@ module Parser
       if !dedent_level.nil?
         dedenter = Lexer::Dedenter.new(dedent_level)
 
-        if node.type == :str
+        case node.type
+        when :str
           str = node.children.first
           dedenter.dedent(str)
-        elsif node.type == :dstr || node.type == :xstr
-          node.children.each do |str_node|
+        when :dstr, :xstr
+          children = node.children.map do |str_node|
             if str_node.type == :str
               str = str_node.children.first
               dedenter.dedent(str)
+              next nil if str.empty?
             else
               dedenter.interrupt
             end
+            str_node
           end
+
+          node = node.updated(nil, children.compact)
         end
       end
 
@@ -557,7 +673,11 @@ module Parser
       when :ident
         name, = *node
 
-        check_assignment_to_numparam(node)
+        var_name = node.children[0].to_s
+        name_loc = node.loc.expression
+
+        check_assignment_to_numparam(var_name, name_loc)
+        check_reserved_for_numparam(var_name, name_loc)
 
         @parser.static_env.declare(name)
 
@@ -619,15 +739,6 @@ module Parser
         binary_op_map(lhs, eql_t, rhs))
     end
 
-    def rassign(lhs, assoc_t, rhs)
-      n(:rasgn, [lhs, rhs], binary_op_map(lhs, assoc_t, rhs))
-    end
-
-    def multi_rassign(lhs, assoc_t, rhs)
-      n(:mrasgn, [ lhs, rhs ],
-        binary_op_map(lhs, assoc_t, rhs))
-    end
-
     #
     # Class and module definition
     #
@@ -657,20 +768,25 @@ module Parser
 
     def def_method(def_t, name_t, args,
                    body, end_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:def, [ value(name_t).to_sym, args, body ],
         definition_map(def_t, nil, name_t, end_t))
     end
 
     def def_endless_method(def_t, name_t, args,
                            assignment_t, body)
-      n(:def_e, [ value(name_t).to_sym, args, body ],
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
+      n(:def, [ value(name_t).to_sym, args, body ],
         endless_definition_map(def_t, nil, name_t, assignment_t, body))
     end
 
     def def_singleton(def_t, definee, dot_t,
                       name_t, args,
                       body, end_t)
-      return unless validate_definee(definee)
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
 
       n(:defs, [ definee, value(name_t).to_sym, args, body ],
         definition_map(def_t, dot_t, name_t, end_t))
@@ -679,9 +795,10 @@ module Parser
     def def_endless_singleton(def_t, definee, dot_t,
                               name_t, args,
                               assignment_t, body)
-      return unless validate_definee(definee)
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
 
-      n(:defs_e, [ definee, value(name_t).to_sym, args, body ],
+      n(:defs, [ definee, value(name_t).to_sym, args, body ],
         endless_definition_map(def_t, dot_t, name_t, assignment_t, body))
     end
 
@@ -709,16 +826,30 @@ module Parser
       n(:numargs, [ max_numparam ], nil)
     end
 
-    def forward_args(begin_t, dots_t, end_t)
-      n(:forward_args, [], collection_map(begin_t, token_map(dots_t), end_t))
+    def forward_only_args(begin_t, dots_t, end_t)
+      if self.class.emit_forward_arg
+        arg = forward_arg(dots_t)
+        n(:args, [ arg ],
+          collection_map(begin_t, [ arg ], end_t))
+      else
+        n(:forward_args, [], collection_map(begin_t, token_map(dots_t), end_t))
+      end
+    end
+
+    def forward_arg(dots_t)
+      n(:forward_arg, [], token_map(dots_t))
     end
 
     def arg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:arg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def optarg(name_t, eql_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:optarg, [ value(name_t).to_sym, value ],
         variable_map(name_t).
           with_operator(loc(eql_t)).
@@ -727,6 +858,7 @@ module Parser
 
     def restarg(star_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
         n(:restarg, [ value(name_t).to_sym ],
           arg_prefix_map(star_t, name_t))
       else
@@ -736,17 +868,23 @@ module Parser
     end
 
     def kwarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwarg, [ value(name_t).to_sym ],
         kwarg_map(name_t))
     end
 
     def kwoptarg(name_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwoptarg, [ value(name_t).to_sym, value ],
         kwarg_map(name_t, value))
     end
 
     def kwrestarg(dstar_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
+
         n(:kwrestarg, [ value(name_t).to_sym ],
           arg_prefix_map(dstar_t, name_t))
       else
@@ -761,11 +899,15 @@ module Parser
     end
 
     def shadowarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:shadowarg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def blockarg(amper_t, name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:blockarg, [ value(name_t).to_sym ],
         arg_prefix_map(amper_t, name_t))
     end
@@ -869,6 +1011,11 @@ module Parser
     def call_method(receiver, dot_t, selector_t,
                     lparen_t=nil, args=[], rparen_t=nil)
       type = call_type_for_dot(dot_t)
+
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       if selector_t.nil?
         n(type, [ receiver, :call, *args ],
           send_map(receiver, dot_t, nil, lparen_t, args, rparen_t))
@@ -946,6 +1093,10 @@ module Parser
     end
 
     def index(receiver, lbrack_t, indexes, rbrack_t)
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(indexes)
+      end
+
       if self.class.emit_index
         n(:index, [ receiver, *indexes ],
           index_map(receiver, lbrack_t, rbrack_t))
@@ -1108,6 +1259,10 @@ module Parser
         end
       end
 
+      if %i[yield super].include?(type) && self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       n(type, args,
         keyword_map(keyword_t, lparen_t, args, rparen_t))
     end
@@ -1241,6 +1396,16 @@ module Parser
         binary_op_map(lhs, in_t, rhs))
     end
 
+    def match_pattern(lhs, match_t, rhs)
+      n(:match_pattern, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
+    def match_pattern_p(lhs, match_t, rhs)
+      n(:match_pattern_p, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
     def in_pattern(in_t, pattern, guard, then_t, body)
       children = [pattern, guard, body]
       n(:in_pattern, children,
@@ -1358,6 +1523,11 @@ module Parser
         collection_map(lbrack_t, elements, rbrack_t))
     end
 
+    def find_pattern(lbrack_t, elements, rbrack_t)
+      n(:find_pattern, elements,
+        collection_map(lbrack_t, elements, rbrack_t))
+    end
+
     def match_with_trailing_comma(match, comma_t)
       n(:match_with_trailing_comma, [ match ], expr_map(match.loc.expression.join(loc(comma_t))))
     end
@@ -1513,8 +1683,10 @@ module Parser
       end
     end
 
-    def check_assignment_to_numparam(node)
-      name = node.children[0].to_s
+    def check_assignment_to_numparam(name, loc)
+      # MRI < 2.7 treats numbered parameters as regular variables
+      # and so it's allowed to perform assignments like `_1 = 42`.
+      return if @parser.version < 27
 
       assigning_to_numparam =
         @parser.context.in_dynamic_block? &&
@@ -1522,7 +1694,17 @@ module Parser
         @parser.max_numparam_stack.has_numparams?
 
       if assigning_to_numparam
-        diagnostic :error, :cant_assign_to_numparam, { :name => name }, node.loc.expression
+        diagnostic :error, :cant_assign_to_numparam, { :name => name }, loc
+      end
+    end
+
+    def check_reserved_for_numparam(name, loc)
+      # MRI < 3.0 accepts assignemnt to variables like _1
+      # if it's not a numbered parameter. MRI 3.0 and newer throws an error.
+      return if @parser.version < 30
+
+      if name =~ /\A_([1-9])\z/
+        diagnostic :error, :reserved_for_numparam, { :name => name }, loc
       end
     end
 
@@ -1756,17 +1938,17 @@ module Parser
     end
 
     def definition_map(keyword_t, operator_t, name_t, end_t)
-      Source::Map::Definition.new(loc(keyword_t),
-                                  loc(operator_t), loc(name_t),
-                                  loc(end_t))
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t),
+                                        loc(end_t), nil, nil)
     end
 
     def endless_definition_map(keyword_t, operator_t, name_t, assignment_t, body_e)
       body_l = body_e.loc.expression
 
-      Source::Map::EndlessDefinition.new(loc(keyword_t),
-                                         loc(operator_t), loc(name_t),
-                                         loc(assignment_t), body_l)
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t), nil,
+                                        loc(assignment_t), body_l)
     end
 
     def send_map(receiver_e, dot_t, selector_t, begin_t=nil, args=[], end_t=nil)
@@ -2023,6 +2205,20 @@ module Parser
         true
       end
     end
+
+    def rewrite_hash_args_to_kwargs(args)
+      if args.any? && kwargs?(args.last)
+        # foo(..., bar: baz)
+        args[args.length - 1] = args[args.length - 1].updated(:kwargs)
+      elsif args.length > 1 && args.last.type == :block_pass && kwargs?(args[args.length - 2])
+        # foo(..., bar: baz, &blk)
+        args[args.length - 2] = args[args.length - 2].updated(:kwargs)
+      end
+    end
+
+    def kwargs?(node)
+      node.type == :hash && node.loc.begin.nil? && node.loc.end.nil?
+    end
   end
 
 end