parser 2.7.2.0 → 3.1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3efc381a3ad92df723e4a698afbceb63f65fdb7c892f690dbd569ba298307d8b
-  data.tar.gz: b9e78eb7dd39f6539fc21883fbec26b1a0c2d377c68bf060d485df5521318e54
+  metadata.gz: 12a38198be7971a87e3fbc2514f45120238a8004b1b267382a481ea971c8d9ec
+  data.tar.gz: ec1a2a51e94eb30280b31e5f26cb28cb1e50b94a9d3a11ae35616b79eaf11b52
 SHA512:
-  metadata.gz: 3c9b2d1df5a463c289f2881c0064c956e88560bc14af30bbb9fe3828ae7a3e1f8a88352172e353b196f34ba1e98640b6bf38bcbb50bef0f8661aacf9e8ccdc1b
-  data.tar.gz: e394d70911b7d86ee7049b0ecaab839120da2f90372630e48c6bb3c14f3b45b35322a2d99ca516df6cc2e8a8e7c0c79aca1031c3dfa28b0026c300ea10b51ed4
+  metadata.gz: e37d66e7c76e0af7d9ad1e9c3c635741436f92d0eef67032fdad83c38090c90cdb9be49018e20786ae540b24d18628695ab4b372a94c96e93b1ce36418e41da5
+  data.tar.gz: 9ad75e610252fb6db157948fb162b0907baedeac7c4107574b78b3869c2490afabec269ab31303ceec2a3a7f17659eb06b43e9b6677b279fb412abade076b93a

data/LICENSE.txt

@@ -0,0 +1,25 @@
+Copyright (c) 2013-2016 whitequark  <whitequark@whitequark.org>
+
+Parts of the source are derived from ruby_parser:
+Copyright (c) Ryan Davis, seattle.rb
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/parser/all.rb

@@ -11,3 +11,5 @@ require 'parser/ruby25'
 require 'parser/ruby26'
 require 'parser/ruby27'
 require 'parser/ruby30'
+require 'parser/ruby31'
+require 'parser/ruby32'

data/lib/parser/ast/processor.rb

@@ -20,6 +20,7 @@ module Parser
       alias on_array    process_regular_node
       alias on_pair     process_regular_node
       alias on_hash     process_regular_node
+      alias on_kwargs   process_regular_node
       alias on_irange   process_regular_node
       alias on_erange   process_regular_node
 
@@ -239,6 +240,8 @@ module Parser
 
       alias on_case_match              process_regular_node
       alias on_in_match                process_regular_node
+      alias on_match_pattern           process_regular_node
+      alias on_match_pattern_p         process_regular_node
       alias on_in_pattern              process_regular_node
       alias on_if_guard                process_regular_node
       alias on_unless_guard            process_regular_node

data/lib/parser/base.rb

@@ -109,6 +109,7 @@ module Parser
     end
     private_class_method :setup_source_buffer
 
+    attr_reader :lexer
     attr_reader :diagnostics
     attr_reader :builder
     attr_reader :static_env

data/lib/parser/builders/default.rb

@@ -128,6 +128,83 @@ module Parser
 
     @emit_forward_arg = false
 
+    class << self
+      ##
+      # AST compatibility attribute; Starting from Ruby 2.7 keyword arguments
+      # of method calls that are passed explicitly as a hash (i.e. with curly braces)
+      # are treated as positional arguments and Ruby 2.7 emits a warning on such method
+      # call. Ruby 3.0 given an ArgumentError.
+      #
+      # If set to false (the default) the last hash argument is emitted as `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (hash
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # If set to true it is emitted as `kwargs`:
+      #
+      # ```
+      # (send nil :foo
+      #   (kwargs
+      #     (pair
+      #       (sym :bar)
+      #       (int 42))))
+      # ```
+      #
+      # Note that `kwargs` node is just a replacement for `hash` argument,
+      # so if there's are multiple arguments (or a `kwsplat`) all of them
+      # are wrapped into `kwargs` instead of `hash`:
+      #
+      # ```
+      # (send nil :foo
+      #   (kwargs
+      #     (pair
+      #       (sym :a)
+      #       (int 42))
+      #     (kwsplat
+      #       (send nil :b))
+      #     (pair
+      #       (sym :c)
+      #       (int 10))))
+      # ```
+      attr_accessor :emit_kwargs
+    end
+
+    @emit_kwargs = false
+
+    class << self
+      ##
+      # AST compatibility attribute; Starting from 3.0 Ruby returns
+      # true/false from single-line pattern matching with `in` keyword.
+      #
+      # Before 3.0 there was an exception if given value doesn't match pattern.
+      #
+      # NOTE: This attribute affects only Ruby 2.7 grammar.
+      # 3.0 grammar always emits `match_pattern`/`match_pattern_p`
+      #
+      # If compatibility attribute set to false `foo in bar` is emitted as `in_match`:
+      #
+      # ```
+      # (in-match
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      #
+      # If set to true it's emitted as `match_pattern_p`:
+      # ```
+      # (match-pattern-p
+      #   (send nil :foo)
+      #   (match-var :bar))
+      # ```
+      attr_accessor :emit_match_pattern
+    end
+
+    @emit_match_pattern = false
+
     class << self
       ##
       # @api private
@@ -138,6 +215,8 @@ module Parser
         @emit_index = true
         @emit_arg_inside_procarg0 = true
         @emit_forward_arg = true
+        @emit_kwargs = true
+        @emit_match_pattern = true
       end
     end
 
@@ -310,18 +389,23 @@ module Parser
       if !dedent_level.nil?
         dedenter = Lexer::Dedenter.new(dedent_level)
 
-        if node.type == :str
+        case node.type
+        when :str
           str = node.children.first
           dedenter.dedent(str)
-        elsif node.type == :dstr || node.type == :xstr
-          node.children.each do |str_node|
+        when :dstr, :xstr
+          children = node.children.map do |str_node|
             if str_node.type == :str
               str = str_node.children.first
               dedenter.dedent(str)
+              next nil if str.empty?
             else
               dedenter.interrupt
             end
+            str_node
           end
+
+          node = node.updated(nil, children.compact)
         end
       end
 
@@ -434,12 +518,52 @@ module Parser
       n(:pair, [ key, value ], pair_map)
     end
 
+    def pair_label(key_t)
+      key_l = loc(key_t)
+      value_l = key_l.adjust(end_pos: -1)
+
+      label = value(key_t)
+      value =
+        if label =~ /\A[[:lower:]]/
+          n(:ident, [ label.to_sym ], Source::Map::Variable.new(value_l))
+        else
+          n(:const, [ nil, label.to_sym ], Source::Map::Constant.new(nil, value_l, value_l))
+        end
+      pair_keyword(key_t, accessible(value))
+    end
+
     def kwsplat(dstar_t, arg)
       n(:kwsplat, [ arg ],
         unary_op_map(dstar_t, arg))
     end
 
     def associate(begin_t, pairs, end_t)
+      0.upto(pairs.length - 1) do |i|
+        (i + 1).upto(pairs.length - 1) do |j|
+          key1, = *pairs[i]
+          key2, = *pairs[j]
+
+          do_warn = false
+
+          # keys have to be simple nodes, MRI ignores equal composite keys like
+          # `{ a(1) => 1, a(1) => 1 }`
+          case key1.type
+          when :sym, :str, :int, :float
+            if key1 == key2
+              do_warn = true
+            end
+          when :rational, :complex, :regexp
+            if @parser.version >= 31 && key1 == key2
+              do_warn = true
+            end
+          end
+
+          if do_warn
+            diagnostic :warning, :duplicate_hash_key, nil, key2.loc.expression
+          end
+        end
+      end
+
       n(:hash, [ *pairs ],
         collection_map(begin_t, pairs, end_t))
     end
@@ -524,19 +648,29 @@ module Parser
       when :ident
         name, = *node
 
-        if @parser.static_env.declared?(name)
-          if name.to_s == parser.current_arg_stack.top
-            diagnostic :error, :circular_argument_reference,
-                       { :var_name => name.to_s }, node.loc.expression
-          end
+        if %w[? !].any? { |c| name.to_s.end_with?(c) }
+          diagnostic :error, :invalid_id_to_get,
+                     { :identifier => name.to_s }, node.loc.expression
+        end
 
-          node.updated(:lvar)
-        else
-          name, = *node
-          n(:send, [ nil, name ],
+        # Numbered parameters are not declared anywhere,
+        # so they take precedence over method calls in numblock contexts
+        if @parser.version >= 27 && @parser.try_declare_numparam(node)
+          return node.updated(:lvar)
+        end
+
+        unless @parser.static_env.declared?(name)
+          return n(:send, [ nil, name ],
             var_send_map(node))
         end
 
+        if name.to_s == parser.current_arg_stack.top
+          diagnostic :error, :circular_argument_reference,
+                     { :var_name => name.to_s }, node.loc.expression
+        end
+
+        node.updated(:lvar)
+
       else
         node
       end
@@ -580,7 +714,7 @@ module Parser
         node.updated(:gvasgn)
 
       when :const
-        unless @parser.context.dynamic_const_definition_allowed?
+        if @parser.context.in_def
           diagnostic :error, :dynamic_const, nil, node.loc.expression
         end
 
@@ -599,6 +733,17 @@ module Parser
 
         node.updated(:lvasgn)
 
+      when :match_var
+        name, = *node
+
+        var_name = node.children[0].to_s
+        name_loc = node.loc.expression
+
+        check_assignment_to_numparam(var_name, name_loc)
+        check_reserved_for_numparam(var_name, name_loc)
+
+        node
+
       when :nil, :self, :true, :false,
            :__FILE__, :__LINE__, :__ENCODING__
         diagnostic :error, :invalid_assignment, nil, node.loc.expression
@@ -655,14 +800,6 @@ module Parser
         binary_op_map(lhs, eql_t, rhs))
     end
 
-    def rassign(lhs, assoc_t, rhs)
-      assign(rhs, assoc_t, lhs)
-    end
-
-    def multi_rassign(lhs, assoc_t, rhs)
-      multi_assign(rhs, assoc_t, lhs)
-    end
-
     #
     # Class and module definition
     #
@@ -742,8 +879,14 @@ module Parser
 
     def args(begin_t, args, end_t, check_args=true)
       args = check_duplicate_args(args) if check_args
-      n(:args, args,
-        collection_map(begin_t, args, end_t))
+      validate_no_forward_arg_after_restarg(args)
+
+      map = collection_map(begin_t, args, end_t)
+      if !self.class.emit_forward_arg && args.length == 1 && args[0].type == :forward_arg
+        n(:forward_args, [], map)
+      else
+        n(:args, args, map)
+      end
     end
 
     def numargs(max_numparam)
@@ -830,9 +973,12 @@ module Parser
     end
 
     def blockarg(amper_t, name_t)
-      check_reserved_for_numparam(value(name_t), loc(name_t))
+      if !name_t.nil?
+        check_reserved_for_numparam(value(name_t), loc(name_t))
+      end
 
-      n(:blockarg, [ value(name_t).to_sym ],
+      arg_name = name_t ? value(name_t).to_sym : nil
+      n(:blockarg, [ arg_name ],
         arg_prefix_map(amper_t, name_t))
     end
 
@@ -935,6 +1081,11 @@ module Parser
     def call_method(receiver, dot_t, selector_t,
                     lparen_t=nil, args=[], rparen_t=nil)
       type = call_type_for_dot(dot_t)
+
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       if selector_t.nil?
         n(type, [ receiver, :call, *args ],
           send_map(receiver, dot_t, nil, lparen_t, args, rparen_t))
@@ -1012,6 +1163,10 @@ module Parser
     end
 
     def index(receiver, lbrack_t, indexes, rbrack_t)
+      if self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(indexes)
+      end
+
       if self.class.emit_index
         n(:index, [ receiver, *indexes ],
           index_map(receiver, lbrack_t, rbrack_t))
@@ -1174,6 +1329,10 @@ module Parser
         end
       end
 
+      if %i[yield super].include?(type) && self.class.emit_kwargs
+        rewrite_hash_args_to_kwargs(args)
+      end
+
       n(type, args,
         keyword_map(keyword_t, lparen_t, args, rparen_t))
     end
@@ -1307,6 +1466,16 @@ module Parser
         binary_op_map(lhs, in_t, rhs))
     end
 
+    def match_pattern(lhs, match_t, rhs)
+      n(:match_pattern, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
+    def match_pattern_p(lhs, match_t, rhs)
+      n(:match_pattern_p, [lhs, rhs],
+        binary_op_map(lhs, match_t, rhs))
+    end
+
     def in_pattern(in_t, pattern, guard, then_t, body)
       children = [pattern, guard, body]
       n(:in_pattern, children,
@@ -1584,6 +1753,21 @@ module Parser
       end
     end
 
+    def validate_no_forward_arg_after_restarg(args)
+      restarg = nil
+      forward_arg = nil
+      args.each do |arg|
+        case arg.type
+        when :restarg then restarg = arg
+        when :forward_arg then forward_arg = arg
+        end
+      end
+
+      if !forward_arg.nil? && !restarg.nil?
+        diagnostic :error, :forward_arg_after_restarg, nil, forward_arg.loc.expression, [restarg.loc.expression]
+      end
+    end
+
     def check_assignment_to_numparam(name, loc)
       # MRI < 2.7 treats numbered parameters as regular variables
       # and so it's allowed to perform assignments like `_1 = 42`.
@@ -1601,7 +1785,7 @@ module Parser
 
     def check_reserved_for_numparam(name, loc)
       # MRI < 3.0 accepts assignemnt to variables like _1
-      # if it's not a numbererd parameter. MRI 3.0 and newer throws an error.
+      # if it's not a numbered parameter. MRI 3.0 and newer throws an error.
       return if @parser.version < 30
 
       if name =~ /\A_([1-9])\z/
@@ -1625,7 +1809,7 @@ module Parser
     end
 
     def check_lvar_name(name, loc)
-      if name =~ /\A[[[:lower:]]|_][[[:alnum:]]_]*\z/
+      if name =~ /\A[[[:lower:]]_][[[:alnum:]]_]*\z/
         # OK
       else
         diagnostic :error, :lvar_name, { name: name }, loc
@@ -2106,6 +2290,20 @@ module Parser
         true
       end
     end
+
+    def rewrite_hash_args_to_kwargs(args)
+      if args.any? && kwargs?(args.last)
+        # foo(..., bar: baz)
+        args[args.length - 1] = args[args.length - 1].updated(:kwargs)
+      elsif args.length > 1 && args.last.type == :block_pass && kwargs?(args[args.length - 2])
+        # foo(..., bar: baz, &blk)
+        args[args.length - 2] = args[args.length - 2].updated(:kwargs)
+      end
+    end
+
+    def kwargs?(node)
+      node.type == :hash && node.loc.begin.nil? && node.loc.end.nil?
+    end
   end
 
 end

data/lib/parser/context.rb

@@ -9,56 +9,41 @@ module Parser
   # + :sclass - in the singleton class body (class << obj; end)
   # + :def - in the method body (def m; end)
   # + :defs - in the singleton method body (def self.m; end)
+  # + :def_open_args - in the arglist of the method definition
+  #                    keep in mind that it's set **only** after reducing the first argument,
+  #                    if you need to handle the first argument check `lex_state == expr_fname`
   # + :block - in the block body (tap {})
   # + :lambda - in the lambda body (-> {})
   #
   class Context
-    attr_reader :stack
+    FLAGS = %i[
+      in_defined
+      in_kwarg
+      in_argdef
+      in_def
+      in_class
+      in_block
+      in_lambda
+    ]
 
     def initialize
-      @stack = []
-      freeze
-    end
-
-    def push(state)
-      @stack << state
-    end
-
-    def pop
-      @stack.pop
+      reset
     end
 
     def reset
-      @stack.clear
-    end
-
-    def in_class?
-      @stack.last == :class
-    end
-
-    def indirectly_in_def?
-      @stack.include?(:def) || @stack.include?(:defs)
-    end
-
-    def class_definition_allowed?
-      def_index = stack.rindex { |item| [:def, :defs].include?(item) }
-      sclass_index = stack.rindex(:sclass)
-
-      def_index.nil? || (!sclass_index.nil? && sclass_index > def_index)
+      @in_defined = false
+      @in_kwarg = false
+      @in_argdef = false
+      @in_def = false
+      @in_class = false
+      @in_block = false
+      @in_lambda = false
     end
-    alias module_definition_allowed? class_definition_allowed?
-    alias dynamic_const_definition_allowed? class_definition_allowed?
 
-    def in_block?
-      @stack.last == :block
-    end
-
-    def in_lambda?
-      @stack.last == :lambda
-    end
+    attr_accessor(*FLAGS)
 
     def in_dynamic_block?
-      in_block? || in_lambda?
+      in_block || in_lambda
     end
   end
 end

data/lib/parser/current.rb

@@ -3,9 +3,9 @@
 module Parser
   class << self
     def warn_syntax_deviation(feature, version)
-      warn "warning: parser/current is loading #{feature}, which recognizes"
-      warn "warning: #{version}-compliant syntax, but you are running #{RUBY_VERSION}."
-      warn "warning: please see https://github.com/whitequark/parser#compatibility-with-ruby-mri."
+      warn "warning: parser/current is loading #{feature}, which recognizes" \
+        "#{version}-compliant syntax, but you are running #{RUBY_VERSION}.\n" \
+        "Please see https://github.com/whitequark/parser#compatibility-with-ruby-mri."
     end
     private :warn_syntax_deviation
   end
@@ -57,7 +57,7 @@ module Parser
     CurrentRuby = Ruby24
 
   when /^2\.5\./
-    current_version = '2.5.8'
+    current_version = '2.5.9'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby25', current_version
     end
@@ -66,7 +66,7 @@ module Parser
     CurrentRuby = Ruby25
 
   when /^2\.6\./
-    current_version = '2.6.6'
+    current_version = '2.6.9'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby26', current_version
     end
@@ -75,7 +75,7 @@ module Parser
     CurrentRuby = Ruby26
 
   when /^2\.7\./
-    current_version = '2.7.2'
+    current_version = '2.7.5'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby27', current_version
     end
@@ -84,7 +84,7 @@ module Parser
     CurrentRuby = Ruby27
 
   when /^3\.0\./
-    current_version = '3.0.0-dev'
+    current_version = '3.0.3'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby30', current_version
     end
@@ -92,10 +92,28 @@ module Parser
     require 'parser/ruby30'
     CurrentRuby = Ruby30
 
+  when /^3\.1\./
+    current_version = '3.1.1'
+    if RUBY_VERSION != current_version
+      warn_syntax_deviation 'parser/ruby31', current_version
+    end
+
+    require 'parser/ruby31'
+    CurrentRuby = Ruby31
+
+  when /^3\.2\./
+    current_version = '3.2.0-dev'
+    if RUBY_VERSION != current_version
+      warn_syntax_deviation 'parser/ruby32', current_version
+    end
+
+    require 'parser/ruby32'
+    CurrentRuby = Ruby32
+
   else # :nocov:
     # Keep this in sync with released Ruby.
-    warn_syntax_deviation 'parser/ruby27', '2.7.x'
-    require 'parser/ruby27'
-    CurrentRuby = Ruby27
+    warn_syntax_deviation 'parser/ruby31', '3.1.x'
+    require 'parser/ruby31'
+    CurrentRuby = Ruby31
   end
 end

data/lib/parser/current_arg_stack.rb

@@ -19,13 +19,16 @@ module Parser
       freeze
     end
 
+    def empty?
+      @stack.size == 0
+    end
+
     def push(value)
       @stack << value
     end
 
     def set(value)
-      pop
-      push(value)
+      @stack[@stack.length - 1] = value
     end
 
     def pop

data/lib/parser/lexer/dedenter.rb

@@ -38,7 +38,13 @@ module Parser
       # Prevent the following error when processing binary encoded source.
       # "\xC0".split # => ArgumentError (invalid byte sequence in UTF-8)
       lines = string.force_encoding(Encoding::BINARY).split("\\\n")
-      lines.map! {|s| s.force_encoding(original_encoding) }
+      if lines.length == 1
+        # If the line continuation sequence was found but there is no second
+        # line, it was not really a line continuation and must be ignored.
+        lines = [string.force_encoding(original_encoding)]
+      else
+        lines.map! {|s| s.force_encoding(original_encoding) }
+      end
 
       if @at_line_begin
         lines_to_dedent = lines

data/lib/parser/lexer/explanation.rb

@@ -18,7 +18,7 @@ module Parser
     def advance
       type, (val, range) = advance_before_explanation
 
-      more = "(in-kwarg)" if @in_kwarg
+      more = "(in-kwarg)" if @context.in_kwarg
 
       puts decorate(range,
                     Color.green("#{type} #{val.inspect}"),