parser 2.7.1.1 → 3.0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb7c9797808fcd8920ef34ccce6dae7f416d0fe3084a863fa3fec7fa30b086fc
-  data.tar.gz: 828a937688a35e277c7d435197ca2d023458de366a8e94fbaa67bdd6d3a04145
+  metadata.gz: 0b1e9fbbc406eb2665aa17f066247e6584182ebe365b3f1d2ace35628f3f12c6
+  data.tar.gz: 8522d4ff0b299c14cb386156a6f04e7d7828bf8c33ed1331a22829ff873312c1
 SHA512:
-  metadata.gz: 679bc103f3d8b2663a1ec2de6c3acac41c90c1d738b40bf4b5ed945b8e3f51e22359caa0789e55009f3bd2c45ac832d00182db9ba9be1f43b18e6fe0e0508278
-  data.tar.gz: b4f4d29766e23ca406644fb578e2134f28a9983d765d4aa45117dcb5a9ee012f27c2b8b65eccf11b7e3818b1543a62cc0483aa510a51352bc2260ad0fd650acd
+  metadata.gz: 5871c263e07cdd6bf88f661419578977f9ddd9e01e1631bab14ff7fb485f466a7b8dacebcda30514d7d1f3c6746b8d2e1f6c6feb7de57a899a1cacfb728c05af
+  data.tar.gz: 4decb43386adfd5be04356d3c9667c1880fa974ad14bc190312bf25a379fad92e4353b0812bcb8810e1863892f14a0daaf6d0c5d31523e2a45a1bb35f65c4ab0

data/lib/parser.rb

@@ -46,6 +46,7 @@ module Parser
     require 'parser/source/map/variable'
     require 'parser/source/map/keyword'
     require 'parser/source/map/definition'
+    require 'parser/source/map/method_definition'
     require 'parser/source/map/send'
     require 'parser/source/map/index'
     require 'parser/source/map/condition'

data/lib/parser/all.rb

@@ -10,3 +10,5 @@ require 'parser/ruby24'
 require 'parser/ruby25'
 require 'parser/ruby26'
 require 'parser/ruby27'
+require 'parser/ruby30'
+require 'parser/ruby31'

data/lib/parser/ast/processor.rb

@@ -20,6 +20,7 @@ module Parser
       alias on_array    process_regular_node
       alias on_pair     process_regular_node
       alias on_hash     process_regular_node
+      alias on_kwargs   process_regular_node
       alias on_irange   process_regular_node
       alias on_erange   process_regular_node
 
@@ -124,6 +125,7 @@ module Parser
       alias on_kwarg          process_argument_node
       alias on_kwoptarg       process_argument_node
       alias on_kwrestarg      process_argument_node
+      alias on_forward_arg    process_argument_node
 
       def on_procarg0(node)
         if node.children[0].is_a?(Symbol)
@@ -238,6 +240,8 @@ module Parser
 
       alias on_case_match              process_regular_node
       alias on_in_match                process_regular_node
+      alias on_match_pattern           process_regular_node
+      alias on_match_pattern_p         process_regular_node
       alias on_in_pattern              process_regular_node
       alias on_if_guard                process_regular_node
       alias on_unless_guard            process_regular_node
@@ -250,6 +254,7 @@ module Parser
       alias on_array_pattern_with_tail process_regular_node
       alias on_hash_pattern            process_regular_node
       alias on_const_pattern           process_regular_node
+      alias on_find_pattern            process_regular_node
 
       # @private
       def process_variable_node(node)

data/lib/parser/base.rb

@@ -109,6 +109,7 @@ module Parser
     end
     private_class_method :setup_source_buffer
 
+    attr_reader :lexer
     attr_reader :diagnostics
     attr_reader :builder
     attr_reader :static_env
@@ -177,16 +178,16 @@ module Parser
     end
 
     ##
-    # Parses a source buffer and returns the AST.
+    # Parses a source buffer and returns the AST, or `nil` in case of a non fatal error.
     #
     # @param [Parser::Source::Buffer] source_buffer The source buffer to parse.
-    # @return [Parser::AST::Node]
+    # @return [Parser::AST::Node, nil]
     #
     def parse(source_buffer)
       @lexer.source_buffer = source_buffer
       @source_buffer       = source_buffer
 
-      do_parse
+      do_parse || nil # Force `false` to `nil`, see https://github.com/ruby/racc/pull/136
     ensure
       # Don't keep references to the source file.
       @source_buffer       = nil
@@ -210,8 +211,9 @@ module Parser
 
     ##
     # Parses a source buffer and returns the AST, the source code comments,
-    # and the tokens emitted by the lexer. If `recover` is true and a fatal
-    # {SyntaxError} is encountered, `nil` is returned instead of the AST, and
+    # and the tokens emitted by the lexer. In case of a fatal error, a {SyntaxError}
+    # is raised, unless `recover` is true. In case of an error
+    # (non-fatal or recovered), `nil` is returned instead of the AST, and
     # comments as well as tokens are only returned up to the location of
     # the error.
     #

data/lib/parser/builders/default.rb

@@ -80,6 +80,8 @@ module Parser
       attr_accessor :emit_index
     end
 
+    @emit_index = false
+
     class << self
       ##
       # AST compatibility attribute; causes a single non-mlhs
@@ -95,7 +97,113 @@ module Parser
       attr_accessor :emit_arg_inside_procarg0
     end
 
-    @emit_index = false
+    @emit_arg_inside_procarg0 = false
+
+    class << self
+      ##
+      # AST compatibility attribute; arguments forwarding initially
+      # didn't have support for leading arguments
+      # (i.e. `def m(a, ...); end` was a syntax error). However, Ruby 3.0
+      # added support for any number of arguments in front of the `...`.
+      #
+      # If set to false (the default):
+      #   1. `def m(...) end` is emitted as
+      #      s(:def, :m, s(:forward_args), nil)
+      #   2. `def m(a, b, ...) end` is emitted as
+      #      s(:def, :m,
+      #        s(:args, s(:arg, :a), s(:arg, :b), s(:forward_arg)))
+      #
+      # If set to true it uses a single format:
+      #   1. `def m(...) end` is emitted as
+      #      s(:def, :m, s(:args, s(:forward_arg)))
+      #   2. `def m(a, b, ...) end` is emitted as
+      #      s(:def, :m, s(:args, s(:arg, :a), s(:arg, :b), s(:forward_arg)))
+      #
+      # It does't matter that much on 2.7 (because there can't be any leading arguments),
+      # but on 3.0 it should be better enabled to use a single AST format.
+      #
+      # @return [Boolean]
+      attr_accessor :emit_forward_arg
+    end
+
+    @emit_forward_arg = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from Ruby 2.7 keyword arguments
+      # of method calls that are passed explicitly as a hash (i.e. with curly braces)
+      # are treated as positional arguments and Ruby 2.7 emits a warning on such method
+      # call. Ruby 3.0 given an ArgumentError.
+      #
+      # If set to false (the default) the last hash argument is emitted as `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # If set to true it is emitted as `kwargs`:
+      #
+      # ```
+      # (send nil :foo
+      #   (kwargs
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # Note that `kwargs` node is just a replacement for `hash` argument,
+      # so if there's are multiple arguments (or a `kwsplat`) all of them
+      # are wrapped into `kwargs` instead of `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :a)
+      #       (int 42))
+      #     (kwsplat
+      #       (send nil :b))
+      #     (pair
+      #       (sym :c)
+      #       (int 10))))
+      # ```
+      attr_accessor :emit_kwargs
+    end
+
+    @emit_kwargs = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from 3.0 Ruby returns
+      # true/false from single-line pattern matching with `in` keyword.
+      #
+      # Before 3.0 there was an exception if given value doesn't match pattern.
+      #
+      # NOTE: This attribute affects only Ruby 2.7 grammar.
+      # 3.0 grammar always emits `match_pattern`/`match_pattern_p`
+      #
+      # If compatibility attribute set to false `foo in bar` is emitted as `in_match`:
+      #
+      # ```
+      # (in-match
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      #
+      # If set to true it's emitted as `match_pattern_p`:
+      # ```
+      # (match-pattern-p
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      attr_accessor :emit_match_pattern
+    end
+
+    @emit_match_pattern = false
 
     class << self
       ##
@@ -106,6 +214,9 @@ module Parser
         @emit_encoding = true
         @emit_index = true
         @emit_arg_inside_procarg0 = true
+        @emit_forward_arg = true
+        @emit_kwargs = true
+        @emit_match_pattern = true
       end
     end
 
@@ -278,18 +389,23 @@ module Parser
       if !dedent_level.nil?
         dedenter = Lexer::Dedenter.new(dedent_level)
 
-        if node.type == :str
+        case node.type
+        when :str
           str = node.children.first
           dedenter.dedent(str)
-        elsif node.type == :dstr || node.type == :xstr
-          node.children.each do |str_node|
+        when :dstr, :xstr
+          children = node.children.map do |str_node|
             if str_node.type == :str
               str = str_node.children.first
               dedenter.dedent(str)
+              next nil if str.empty?
             else
               dedenter.interrupt
             end
+            str_node
           end
+
+          node = node.updated(nil, children.compact)
         end
       end
 
@@ -557,7 +673,11 @@ module Parser
       when :ident
         name, = *node
 
-        check_assignment_to_numparam(node)
+        var_name = node.children[0].to_s
+        name_loc = node.loc.expression
+
+        check_assignment_to_numparam(var_name, name_loc)
+        check_reserved_for_numparam(var_name, name_loc)
 
         @parser.static_env.declare(name)
 
@@ -648,23 +768,38 @@ module Parser
 
     def def_method(def_t, name_t, args,
                    body, end_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:def, [ value(name_t).to_sym, args, body ],
         definition_map(def_t, nil, name_t, end_t))
     end
 
+    def def_endless_method(def_t, name_t, args,
+                           assignment_t, body)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
+      n(:def, [ value(name_t).to_sym, args, body ],
+        endless_definition_map(def_t, nil, name_t, assignment_t, body))
+    end
+
     def def_singleton(def_t, definee, dot_t,
                       name_t, args,
                       body, end_t)
-      case definee.type
-      when :int, :str, :dstr, :sym, :dsym,
-           :regexp, :array, :hash
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
 
-        diagnostic :error, :singleton_literal, nil, definee.loc.expression
+      n(:defs, [ definee, value(name_t).to_sym, args, body ],
+        definition_map(def_t, dot_t, name_t, end_t))
+    end
 
-      else
-        n(:defs, [ definee, value(name_t).to_sym, args, body ],
-          definition_map(def_t, dot_t, name_t, end_t))
-      end
+    def def_endless_singleton(def_t, definee, dot_t,
+                              name_t, args,
+                              assignment_t, body)
+      validate_definee(definee)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
+      n(:defs, [ definee, value(name_t).to_sym, args, body ],
+        endless_definition_map(def_t, dot_t, name_t, assignment_t, body))
     end
 
     def undef_method(undef_t, names)
@@ -691,16 +826,30 @@ module Parser
       n(:numargs, [ max_numparam ], nil)
     end
 
-    def forward_args(begin_t, dots_t, end_t)
-      n(:forward_args, [], collection_map(begin_t, token_map(dots_t), end_t))
+    def forward_only_args(begin_t, dots_t, end_t)
+      if self.class.emit_forward_arg
+        arg = forward_arg(dots_t)
+        n(:args, [ arg ],
+          collection_map(begin_t, [ arg ], end_t))
+      else
+        n(:forward_args, [], collection_map(begin_t, token_map(dots_t), end_t))
+      end
+    end
+
+    def forward_arg(dots_t)
+      n(:forward_arg, [], token_map(dots_t))
     end
 
     def arg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:arg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def optarg(name_t, eql_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:optarg, [ value(name_t).to_sym, value ],
         variable_map(name_t).
           with_operator(loc(eql_t)).
@@ -709,6 +858,7 @@ module Parser
 
     def restarg(star_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
         n(:restarg, [ value(name_t).to_sym ],
           arg_prefix_map(star_t, name_t))
       else
@@ -718,17 +868,23 @@ module Parser
     end
 
     def kwarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwarg, [ value(name_t).to_sym ],
         kwarg_map(name_t))
     end
 
     def kwoptarg(name_t, value)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:kwoptarg, [ value(name_t).to_sym, value ],
         kwarg_map(name_t, value))
     end
 
     def kwrestarg(dstar_t, name_t=nil)
       if name_t
+        check_reserved_for_numparam(value(name_t), loc(name_t))
+
         n(:kwrestarg, [ value(name_t).to_sym ],
           arg_prefix_map(dstar_t, name_t))
       else
@@ -743,11 +899,15 @@ module Parser
     end
 
     def shadowarg(name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:shadowarg, [ value(name_t).to_sym ],
         variable_map(name_t))
     end
 
     def blockarg(amper_t, name_t)
+      check_reserved_for_numparam(value(name_t), loc(name_t))
+
       n(:blockarg, [ value(name_t).to_sym ],
         arg_prefix_map(amper_t, name_t))
     end
@@ -851,6 +1011,11 @@ module Parser
     def call_method(receiver, dot_t, selector_t,
                     lparen_t=nil, args=[], rparen_t=nil)
       type = call_type_for_dot(dot_t)
+
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       if selector_t.nil?
         n(type, [ receiver, :call, *args ],
           send_map(receiver, dot_t, nil, lparen_t, args, rparen_t))
@@ -928,6 +1093,10 @@ module Parser
     end
 
     def index(receiver, lbrack_t, indexes, rbrack_t)
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(indexes)
+      end
+
       if self.class.emit_index
         n(:index, [ receiver, *indexes ],
           index_map(receiver, lbrack_t, rbrack_t))
@@ -1090,6 +1259,10 @@ module Parser
         end
       end
 
+      if %i[yield super].include?(type) && self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       n(type, args,
         keyword_map(keyword_t, lparen_t, args, rparen_t))
     end
@@ -1223,6 +1396,16 @@ module Parser
         binary_op_map(lhs, in_t, rhs))
     end
 
+    def match_pattern(lhs, match_t, rhs)
+      n(:match_pattern, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
+    def match_pattern_p(lhs, match_t, rhs)
+      n(:match_pattern_p, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
     def in_pattern(in_t, pattern, guard, then_t, body)
       children = [pattern, guard, body]
       n(:in_pattern, children,
@@ -1239,8 +1422,10 @@ module Parser
 
     def match_var(name_t)
       name = value(name_t).to_sym
+      name_l = loc(name_t)
 
-      check_duplicate_pattern_variable(name, loc(name_t))
+      check_lvar_name(name, name_l)
+      check_duplicate_pattern_variable(name, name_l)
       @parser.static_env.declare(name)
 
       n(:match_var, [ name ],
@@ -1253,6 +1438,7 @@ module Parser
       expr_l = loc(name_t)
       name_l = expr_l.adjust(end_pos: -1)
 
+      check_lvar_name(name, name_l)
       check_duplicate_pattern_variable(name, name_l)
       @parser.static_env.declare(name)
 
@@ -1293,6 +1479,9 @@ module Parser
           Source::Map::Variable.new(name_l, expr_l))
       when :begin
         match_hash_var_from_str(begin_t, string.children, end_t)
+      else
+        # we only can get here if there is an interpolation, e.g., ``in "#{ a }":`
+        diagnostic :error, :pm_interp_in_var_name, nil, loc(begin_t).join(loc(end_t))
       end
     end
 
@@ -1334,6 +1523,11 @@ module Parser
         collection_map(lbrack_t, elements, rbrack_t))
     end
 
+    def find_pattern(lbrack_t, elements, rbrack_t)
+      n(:find_pattern, elements,
+        collection_map(lbrack_t, elements, rbrack_t))
+    end
+
     def match_with_trailing_comma(match, comma_t)
       n(:match_with_trailing_comma, [ match ], expr_map(match.loc.expression.join(loc(comma_t))))
     end
@@ -1489,8 +1683,10 @@ module Parser
       end
     end
 
-    def check_assignment_to_numparam(node)
-      name = node.children[0].to_s
+    def check_assignment_to_numparam(name, loc)
+      # MRI < 2.7 treats numbered parameters as regular variables
+      # and so it's allowed to perform assignments like `_1 = 42`.
+      return if @parser.version < 27
 
       assigning_to_numparam =
         @parser.context.in_dynamic_block? &&
@@ -1498,7 +1694,17 @@ module Parser
         @parser.max_numparam_stack.has_numparams?
 
       if assigning_to_numparam
-        diagnostic :error, :cant_assign_to_numparam, { :name => name }, node.loc.expression
+        diagnostic :error, :cant_assign_to_numparam, { :name => name }, loc
+      end
+    end
+
+    def check_reserved_for_numparam(name, loc)
+      # MRI < 3.0 accepts assignemnt to variables like _1
+      # if it's not a numbered parameter. MRI 3.0 and newer throws an error.
+      return if @parser.version < 30
+
+      if name =~ /\A_([1-9])\z/
+        diagnostic :error, :reserved_for_numparam, { :name => name }, loc
       end
     end
 
@@ -1518,7 +1724,7 @@ module Parser
     end
 
     def check_lvar_name(name, loc)
-      if name =~ /\A[[[:lower:]]|_][[[:alnum:]]_]*\z/
+      if name =~ /\A[[[:lower:]]_][[[:alnum:]]_]*\z/
         # OK
       else
         diagnostic :error, :lvar_name, { name: name }, loc
@@ -1732,9 +1938,17 @@ module Parser
     end
 
     def definition_map(keyword_t, operator_t, name_t, end_t)
-      Source::Map::Definition.new(loc(keyword_t),
-                                  loc(operator_t), loc(name_t),
-                                  loc(end_t))
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t),
+                                        loc(end_t), nil, nil)
+    end
+
+    def endless_definition_map(keyword_t, operator_t, name_t, assignment_t, body_e)
+      body_l = body_e.loc.expression
+
+      Source::Map::MethodDefinition.new(loc(keyword_t),
+                                        loc(operator_t), loc(name_t), nil,
+                                        loc(assignment_t), body_l)
     end
 
     def send_map(receiver_e, dot_t, selector_t, begin_t=nil, args=[], end_t=nil)
@@ -1979,6 +2193,32 @@ module Parser
         @parser.send :yyerror
       end
     end
+
+    def validate_definee(definee)
+      case definee.type
+      when :int, :str, :dstr, :sym, :dsym,
+           :regexp, :array, :hash
+
+        diagnostic :error, :singleton_literal, nil, definee.loc.expression
+        false
+      else
+        true
+      end
+    end
+
+    def rewrite_hash_args_to_kwargs(args)
+      if args.any? && kwargs?(args.last)
+        # foo(..., bar: baz)
+        args[args.length - 1] = args[args.length - 1].updated(:kwargs)
+      elsif args.length > 1 && args.last.type == :block_pass && kwargs?(args[args.length - 2])
+        # foo(..., bar: baz, &blk)
+        args[args.length - 2] = args[args.length - 2].updated(:kwargs)
+      end
+    end
+
+    def kwargs?(node)
+      node.type == :hash && node.loc.begin.nil? && node.loc.end.nil?
+    end
   end
 
 end