parser 2.7.1.4 → 3.0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4dbd5dfd3a249d65b4d2b20c54b668749b3d5360e7feef7772924c135b78d86
-  data.tar.gz: ca30aca7f92a24fb96ef74c1a21149d9d43fb541d86bdae0b113f5ea5e267137
+  metadata.gz: 6c85d02ca4bf90dcc1d33424dd323ae54b9e60581a6473012e950f82ce15d28e
+  data.tar.gz: ebe2c4eb02779209b8a34e051d717a6af84da81fadfeff79d9eab194ecf95d93
 SHA512:
-  metadata.gz: deb6edb6b828f309ebc6eb58e19613fe13c490e780895dfad01aceeeb0111781bc9e90b81bf6be35069afed44bd5f683c8120f134abd3d6aa7546a15fb370c19
-  data.tar.gz: 0c5a25dade42d9c74bc115aa8ecfd8dbfbcdeadc0fee6e994f44532ed994ac0c47aeccccc5405ddc9b5bd3804e4c47874163e6d468e47cb14ddd647ffb2c118c
+  metadata.gz: 173d8e1bbbbbf0ef06d06f1ea59dc41ca49c8b39c6931523dc70bac259eee1e9c9263730d560afe106c70f88f01ae14edc77862d20a4924a674131ca90b9807c
+  data.tar.gz: 1698922a4ff14bf3e826f10b826a8a8e0c3aaf7fefd20669a4666c0ed6ea8a3ed405acbb28fe90985510dedff6414c0330d5affa4639840178bf20dee561e4a9

data/lib/parser.rb

@@ -46,7 +46,7 @@ module Parser
     require 'parser/source/map/variable'
     require 'parser/source/map/keyword'
     require 'parser/source/map/definition'
-    require 'parser/source/map/endless_definition'
+    require 'parser/source/map/method_definition'
     require 'parser/source/map/send'
     require 'parser/source/map/index'
     require 'parser/source/map/condition'

data/lib/parser/all.rb

@@ -10,4 +10,5 @@ require 'parser/ruby24'
 require 'parser/ruby25'
 require 'parser/ruby26'
 require 'parser/ruby27'
-require 'parser/ruby28'
+require 'parser/ruby30'
+require 'parser/ruby31'

data/lib/parser/ast/processor.rb

@@ -20,6 +20,7 @@ module Parser
       alias on_array    process_regular_node
       alias on_pair     process_regular_node
       alias on_hash     process_regular_node
+      alias on_kwargs   process_regular_node
       alias on_irange   process_regular_node
       alias on_erange   process_regular_node
 
@@ -75,9 +76,6 @@ module Parser
       alias on_mlhs     process_regular_node
       alias on_masgn    process_regular_node
 
-      alias on_rasgn    process_regular_node
-      alias on_mrasgn   process_regular_node
-
       def on_const(node)
         scope_node, name = *node
 
@@ -163,8 +161,6 @@ module Parser
         ])
       end
 
-      alias on_def_e on_def
-
       def on_defs(node)
         definee_node, name, args_node, body_node = *node
 
@@ -174,8 +170,6 @@ module Parser
         ])
       end
 
-      alias on_defs_e on_defs
-
       alias on_undef    process_regular_node
       alias on_alias    process_regular_node
 
@@ -246,6 +240,8 @@ module Parser
 
       alias on_case_match              process_regular_node
       alias on_in_match                process_regular_node
+      alias on_match_pattern           process_regular_node
+      alias on_match_pattern_p         process_regular_node
       alias on_in_pattern              process_regular_node
       alias on_if_guard                process_regular_node
       alias on_unless_guard            process_regular_node

data/lib/parser/base.rb

@@ -109,6 +109,7 @@ module Parser
     end
     private_class_method :setup_source_buffer
 
+    attr_reader :lexer
     attr_reader :diagnostics
     attr_reader :builder
     attr_reader :static_env
@@ -177,16 +178,16 @@ module Parser
     end
 
     ##
-    # Parses a source buffer and returns the AST.
+    # Parses a source buffer and returns the AST, or `nil` in case of a non fatal error.
     #
     # @param [Parser::Source::Buffer] source_buffer The source buffer to parse.
-    # @return [Parser::AST::Node]
+    # @return [Parser::AST::Node, nil]
     #
     def parse(source_buffer)
       @lexer.source_buffer = source_buffer
       @source_buffer       = source_buffer
 
-      do_parse
+      do_parse || nil # Force `false` to `nil`, see https://github.com/ruby/racc/pull/136
     ensure
       # Don't keep references to the source file.
       @source_buffer       = nil
@@ -210,8 +211,9 @@ module Parser
 
     ##
     # Parses a source buffer and returns the AST, the source code comments,
-    # and the tokens emitted by the lexer. If `recover` is true and a fatal
-    # {SyntaxError} is encountered, `nil` is returned instead of the AST, and
+    # and the tokens emitted by the lexer. In case of a fatal error, a {SyntaxError}
+    # is raised, unless `recover` is true. In case of an error
+    # (non-fatal or recovered), `nil` is returned instead of the AST, and
     # comments as well as tokens are only returned up to the location of
     # the error.
     #

data/lib/parser/builders/default.rb

@@ -103,7 +103,7 @@ module Parser
       ##
       # AST compatibility attribute; arguments forwarding initially
       # didn't have support for leading arguments
-      # (i.e. `def m(a, ...); end` was a syntax error). However, Ruby 2.8
+      # (i.e. `def m(a, ...); end` was a syntax error). However, Ruby 3.0
       # added support for any number of arguments in front of the `...`.
       #
       # If set to false (the default):
@@ -120,7 +120,7 @@ module Parser
       #      s(:def, :m, s(:args, s(:arg, :a), s(:arg, :b), s(:forward_arg)))
       #
       # It does't matter that much on 2.7 (because there can't be any leading arguments),
-      # but on 2.8 it should be better enabled to use a single AST format.
+      # but on 3.0 it should be better enabled to use a single AST format.
       #
       # @return [Boolean]
       attr_accessor :emit_forward_arg
@@ -128,6 +128,83 @@ module Parser
 
     @emit_forward_arg = false
 
+    class << self
+      ##
+      # AST compatibility attribute; Starting from Ruby 2.7 keyword arguments
+      # of method calls that are passed explicitly as a hash (i.e. with curly braces)
+      # are treated as positional arguments and Ruby 2.7 emits a warning on such method
+      # call. Ruby 3.0 given an ArgumentError.
+      #
+      # If set to false (the default) the last hash argument is emitted as `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # If set to true it is emitted as `kwargs`:
+      #
+      # ```
+      # (send nil :foo
+      #   (kwargs
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # Note that `kwargs` node is just a replacement for `hash` argument,
+      # so if there's are multiple arguments (or a `kwsplat`) all of them
+      # are wrapped into `kwargs` instead of `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :a)
+      #       (int 42))
+      #     (kwsplat
+      #       (send nil :b))
+      #     (pair
+      #       (sym :c)
+      #       (int 10))))
+      # ```
+      attr_accessor :emit_kwargs
+    end
+
+    @emit_kwargs = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from 3.0 Ruby returns
+      # true/false from single-line pattern matching with `in` keyword.
+      #
+      # Before 3.0 there was an exception if given value doesn't match pattern.
+      #
+      # NOTE: This attribute affects only Ruby 2.7 grammar.
+      # 3.0 grammar always emits `match_pattern`/`match_pattern_p`
+      #
+      # If compatibility attribute set to false `foo in bar` is emitted as `in_match`:
+      #
+      # ```
+      # (in-match
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      #
+      # If set to true it's emitted as `match_pattern_p`:
+      # ```
+      # (match-pattern-p
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      attr_accessor :emit_match_pattern
+    end
+
+    @emit_match_pattern = false
+
     class << self
       ##
       # @api private
@@ -138,6 +215,8 @@ module Parser
         @emit_index = true
         @emit_arg_inside_procarg0 = true
         @emit_forward_arg = true
+        @emit_kwargs = true
+        @emit_match_pattern = true
       end
     end
 
@@ -310,18 +389,23 @@ module Parser
       if !dedent_level.nil?
         dedenter = Lexer::Dedenter.new(dedent_level)
 
-        if node.type == :str
+        case node.type
+        when :str
           str = node.children.first
           dedenter.dedent(str)
-        elsif node.type == :dstr || node.type == :xstr
-          node.children.each do |str_node|
+        when :dstr, :xstr
+          children = node.children.map do |str_node|
             if str_node.type == :str
               str = str_node.children.first
               dedenter.dedent(str)
+              next nil if str.empty?
             else
               dedenter.interrupt
             end
+            str_node
           end
+
+          node = node.updated(nil, children.compact)
         end
       end
 
@@ -589,7 +673,11 @@ module Parser
       when :ident
         name, = *node
 
-        check_assignment_to_numparam(node)
+        var_name = node.children[0].to_s
+        name_loc = node.loc.expression
+
+        check_assignment_to_numparam(var_name, name_loc)
+        check_reserved_for_numparam(var_name, name_loc)
 
         @parser.static_env.declare(name)
 
@@ -651,15 +739,6 @@ module Parser
         binary_op_map(lhs, eql_t, rhs))
     end
 
-    def rassign(lhs, assoc_t, rhs)
-      n(:rasgn, [lhs, rhs], binary_op_map(lhs, assoc_t, rhs))
-    end
-
-    def multi_rassign(lhs, assoc_t, rhs)
-      n(:mrasgn, [ lhs, rhs ],
-        binary_op_map(lhs, assoc_t, rhs))
-    end
-
     #
     # Class and module definition
     #
@@ -689,20 +768,25 @@ module Parser
 
     def def_method(def_t, name_t, args,
                    body, end_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:def, [ value(name_t).to_sym, args, body ],
         definition_map(def_t, nil, name_t, end_t))
     end
 
     def def_endless_method(def_t, name_t, args,
                            assignment_t, body)
-      n(:def_e, [ value(name_t).to_sym, args, body ],
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
+      n(:def, [ value(name_t).to_sym, args, body ],
         endless_definition_map(def_t, nil, name_t, assignment_t, body))
     end
 
     def def_singleton(def_t, definee, dot_t,
                       name_t, args,
                       body, end_t)
-      return unless validate_definee(definee)
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
 
       n(:defs, [ definee, value(name_t).to_sym, args, body ],
         definition_map(def_t, dot_t, name_t, end_t))
@@ -711,9 +795,10 @@ module Parser
     def def_endless_singleton(def_t, definee, dot_t,
                               name_t, args,
                               assignment_t, body)
-      return unless validate_definee(definee)
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
 
-      n(:defs_e, [ definee, value(name_t).to_sym, args, body ],
+      n(:defs, [ definee, value(name_t).to_sym, args, body ],
         endless_definition_map(def_t, dot_t, name_t, assignment_t, body))
     end
 
@@ -756,11 +841,15 @@ module Parser
     end
 
     def arg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:arg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def optarg(name_t, eql_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:optarg, [ value(name_t).to_sym, value ],
         variable_map(name_t).
           with_operator(loc(eql_t)).
@@ -769,6 +858,7 @@ module Parser
 
     def restarg(star_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
         n(:restarg, [ value(name_t).to_sym ],
           arg_prefix_map(star_t, name_t))
       else
@@ -778,17 +868,23 @@ module Parser
     end
 
     def kwarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwarg, [ value(name_t).to_sym ],
         kwarg_map(name_t))
     end
 
     def kwoptarg(name_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwoptarg, [ value(name_t).to_sym, value ],
         kwarg_map(name_t, value))
     end
 
     def kwrestarg(dstar_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
+
         n(:kwrestarg, [ value(name_t).to_sym ],
           arg_prefix_map(dstar_t, name_t))
       else
@@ -803,11 +899,15 @@ module Parser
     end
 
     def shadowarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:shadowarg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def blockarg(amper_t, name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:blockarg, [ value(name_t).to_sym ],
         arg_prefix_map(amper_t, name_t))
     end
@@ -911,6 +1011,11 @@ module Parser
     def call_method(receiver, dot_t, selector_t,
                     lparen_t=nil, args=[], rparen_t=nil)
       type = call_type_for_dot(dot_t)
+
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       if selector_t.nil?
         n(type, [ receiver, :call, *args ],
           send_map(receiver, dot_t, nil, lparen_t, args, rparen_t))
@@ -988,6 +1093,10 @@ module Parser
     end
 
     def index(receiver, lbrack_t, indexes, rbrack_t)
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(indexes)
+      end
+
       if self.class.emit_index
         n(:index, [ receiver, *indexes ],
           index_map(receiver, lbrack_t, rbrack_t))
@@ -1150,6 +1259,10 @@ module Parser
         end
       end
 
+      if %i[yield super].include?(type) && self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       n(type, args,
         keyword_map(keyword_t, lparen_t, args, rparen_t))
     end
@@ -1283,6 +1396,16 @@ module Parser
         binary_op_map(lhs, in_t, rhs))
     end
 
+    def match_pattern(lhs, match_t, rhs)
+      n(:match_pattern, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
+    def match_pattern_p(lhs, match_t, rhs)
+      n(:match_pattern_p, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
     def in_pattern(in_t, pattern, guard, then_t, body)
       children = [pattern, guard, body]
       n(:in_pattern, children,
@@ -1560,8 +1683,10 @@ module Parser
       end
     end
 
-    def check_assignment_to_numparam(node)
-      name = node.children[0].to_s
+    def check_assignment_to_numparam(name, loc)
+      # MRI < 2.7 treats numbered parameters as regular variables
+      # and so it's allowed to perform assignments like `_1 = 42`.
+      return if @parser.version < 27
 
       assigning_to_numparam =
         @parser.context.in_dynamic_block? &&
@@ -1569,7 +1694,17 @@ module Parser
         @parser.max_numparam_stack.has_numparams?
 
       if assigning_to_numparam
-        diagnostic :error, :cant_assign_to_numparam, { :name => name }, node.loc.expression
+        diagnostic :error, :cant_assign_to_numparam, { :name => name }, loc
+      end
+    end
+
+    def check_reserved_for_numparam(name, loc)
+      # MRI < 3.0 accepts assignemnt to variables like _1
+      # if it's not a numbered parameter. MRI 3.0 and newer throws an error.
+      return if @parser.version < 30
+
+      if name =~ /\A_([1-9])\z/
+        diagnostic :error, :reserved_for_numparam, { :name => name }, loc
       end
     end
 
@@ -1589,7 +1724,7 @@ module Parser
     end
 
     def check_lvar_name(name, loc)
-      if name =~ /\A[[[:lower:]]|_][[[:alnum:]]_]*\z/
+      if name =~ /\A[[[:lower:]]_][[[:alnum:]]_]*\z/
         # OK
       else
         diagnostic :error, :lvar_name, { name: name }, loc
@@ -1803,17 +1938,17 @@ module Parser
     end
 
     def definition_map(keyword_t, operator_t, name_t, end_t)
-      Source::Map::Definition.new(loc(keyword_t),
-                                  loc(operator_t), loc(name_t),
-                                  loc(end_t))
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t),
+                                        loc(end_t), nil, nil)
     end
 
     def endless_definition_map(keyword_t, operator_t, name_t, assignment_t, body_e)
       body_l = body_e.loc.expression
 
-      Source::Map::EndlessDefinition.new(loc(keyword_t),
-                                         loc(operator_t), loc(name_t),
-                                         loc(assignment_t), body_l)
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t), nil,
+                                        loc(assignment_t), body_l)
     end
 
     def send_map(receiver_e, dot_t, selector_t, begin_t=nil, args=[], end_t=nil)
@@ -2070,6 +2205,20 @@ module Parser
         true
       end
     end
+
+    def rewrite_hash_args_to_kwargs(args)
+      if args.any? && kwargs?(args.last)
+        # foo(..., bar: baz)
+        args[args.length - 1] = args[args.length - 1].updated(:kwargs)
+      elsif args.length > 1 && args.last.type == :block_pass && kwargs?(args[args.length - 2])
+        # foo(..., bar: baz, &blk)
+        args[args.length - 2] = args[args.length - 2].updated(:kwargs)
+      end
+    end
+
+    def kwargs?(node)
+      node.type == :hash && node.loc.begin.nil? && node.loc.end.nil?
+    end
   end
 
 end