rubocop-ast 1.24.1 → 1.46.0

data/lib/rubocop/ast/processed_source.rb

@@ -2,40 +2,68 @@
 
 require 'digest/sha1'
 
-# rubocop:disable Metrics/ClassLength
 module RuboCop
   module AST
+    # A `Prism` interface's class that provides a fixed `Prism::ParseLexResult` instead of parsing.
+    #
+    # This class implements the `parse_lex` method to return a preparsed `Prism::ParseLexResult`
+    # rather than parsing the source code.  When the parse result is already available externally,
+    # such as in Ruby LSP, the Prism parsing process can be bypassed.
+    class PrismPreparsed
+      def initialize(prism_result)
+        unless prism_result.is_a?(Prism::ParseLexResult)
+          raise ArgumentError, <<~MESSAGE
+            Expected a `Prism::ParseLexResult` object, but received `#{prism_result.class}`.
+          MESSAGE
+        end
+
+        @prism_result = prism_result
+      end
+
+      def parse_lex(_source, **_prism_options)
+        @prism_result
+      end
+    end
+
     # ProcessedSource contains objects which are generated by Parser
     # and other information such as disabled lines for cops.
     # It also provides a convenient way to access source lines.
-    class ProcessedSource
+    class ProcessedSource # rubocop:disable Metrics/ClassLength
       # @api private
       STRING_SOURCE_NAME = '(string)'
 
       INVALID_LEVELS = %i[error fatal].freeze
       private_constant :INVALID_LEVELS
 
+      PARSER_ENGINES = %i[default parser_whitequark parser_prism].freeze
+      private_constant :PARSER_ENGINES
+
       attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
-                  :parser_error, :raw_source, :ruby_version
+                  :parser_error, :raw_source, :ruby_version, :parser_engine
 
-      def self.from_file(path, ruby_version)
+      def self.from_file(path, ruby_version, parser_engine: :default)
         file = File.read(path, mode: 'rb')
-        new(file, ruby_version, path)
+        new(file, ruby_version, path, parser_engine: parser_engine)
       end
 
-      def initialize(source, ruby_version, path = nil)
+      def initialize(
+        source, ruby_version, path = nil, parser_engine: :default, prism_result: nil
+      )
+        parser_engine = normalize_parser_engine(parser_engine, ruby_version)
+
         # Defaults source encoding to UTF-8, regardless of the encoding it has
         # been read with, which could be non-utf8 depending on the default
         # external encoding.
-        source.force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8
+        (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8
 
         @raw_source = source
         @path = path
         @diagnostics = []
         @ruby_version = ruby_version
+        @parser_engine = parser_engine
         @parser_error = nil
 
-        parse(source, ruby_version)
+        parse(source, ruby_version, parser_engine, prism_result)
       end
 
       def ast_with_comments
@@ -193,13 +221,13 @@ module RuboCop
         end
       end
 
-      def parse(source, ruby_version)
+      def parse(source, ruby_version, parser_engine, prism_result)
         buffer_name = @path || STRING_SOURCE_NAME
         @buffer = Parser::Source::Buffer.new(buffer_name, 1)
 
         begin
           @buffer.source = source
-        rescue EncodingError => e
+        rescue EncodingError, Parser::UnknownEncodingInMagicComment => e
           @parser_error = e
           @ast = nil
           @comments = []
@@ -207,7 +235,9 @@ module RuboCop
           return
         end
 
-        @ast, @comments, @tokens = tokenize(create_parser(ruby_version))
+        parser = create_parser(ruby_version, parser_engine, prism_result)
+
+        @ast, @comments, @tokens = tokenize(parser)
       end
 
       def tokenize(parser)
@@ -226,56 +256,102 @@ module RuboCop
         [ast, comments, tokens]
       end
 
-      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-      def parser_class(ruby_version)
-        case ruby_version
-        when 1.9
-          require 'parser/ruby19'
-          Parser::Ruby19
-        when 2.0
-          require 'parser/ruby20'
-          Parser::Ruby20
-        when 2.1
-          require 'parser/ruby21'
-          Parser::Ruby21
-        when 2.2
-          require 'parser/ruby22'
-          Parser::Ruby22
-        when 2.3
-          require 'parser/ruby23'
-          Parser::Ruby23
-        when 2.4
-          require 'parser/ruby24'
-          Parser::Ruby24
-        when 2.5
-          require 'parser/ruby25'
-          Parser::Ruby25
-        when 2.6
-          require 'parser/ruby26'
-          Parser::Ruby26
-        when 2.7
-          require 'parser/ruby27'
-          Parser::Ruby27
-        when 2.8, 3.0
-          require 'parser/ruby30'
-          Parser::Ruby30
-        when 3.1
-          require 'parser/ruby31'
-          Parser::Ruby31
-        when 3.2
-          require 'parser/ruby32'
-          Parser::Ruby32
-        else
-          raise ArgumentError,
-                "RuboCop found unknown Ruby version: #{ruby_version.inspect}"
+      # rubocop:disable Lint/FloatComparison, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      def parser_class(ruby_version, parser_engine)
+        case parser_engine
+        when :parser_whitequark
+          case ruby_version
+          when 1.9
+            require 'parser/ruby19'
+            Parser::Ruby19
+          when 2.0
+            require 'parser/ruby20'
+            Parser::Ruby20
+          when 2.1
+            require 'parser/ruby21'
+            Parser::Ruby21
+          when 2.2
+            require 'parser/ruby22'
+            Parser::Ruby22
+          when 2.3
+            require 'parser/ruby23'
+            Parser::Ruby23
+          when 2.4
+            require 'parser/ruby24'
+            Parser::Ruby24
+          when 2.5
+            require 'parser/ruby25'
+            Parser::Ruby25
+          when 2.6
+            require 'parser/ruby26'
+            Parser::Ruby26
+          when 2.7
+            require 'parser/ruby27'
+            Parser::Ruby27
+          when 2.8, 3.0
+            require 'parser/ruby30'
+            Parser::Ruby30
+          when 3.1
+            require 'parser/ruby31'
+            Parser::Ruby31
+          when 3.2
+            require 'parser/ruby32'
+            Parser::Ruby32
+          when 3.3
+            require 'parser/ruby33'
+            Parser::Ruby33
+          when 3.4
+            require 'parser/ruby34'
+            Parser::Ruby34
+          else
+            raise ArgumentError, 'RuboCop supports target Ruby versions 3.4 and below with ' \
+                                 "`parser`. Specified target Ruby version: #{ruby_version.inspect}"
+          end
+        when :parser_prism
+          case ruby_version
+          when 3.3
+            require 'prism/translation/parser33'
+            Prism::Translation::Parser33
+          when 3.4
+            require 'prism/translation/parser34'
+            Prism::Translation::Parser34
+          when 3.5
+            require 'prism/translation/parser35'
+            Prism::Translation::Parser35
+          else
+            raise ArgumentError, 'RuboCop supports target Ruby versions 3.3 and above with Prism. ' \
+                                 "Specified target Ruby version: #{ruby_version.inspect}"
+          end
         end
       end
-      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      # rubocop:enable Lint/FloatComparison, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
 
-      def create_parser(ruby_version)
-        builder = RuboCop::AST::Builder.new
+      def builder_class(parser_engine)
+        case parser_engine
+        when :parser_whitequark
+          RuboCop::AST::Builder
+        when :parser_prism
+          RuboCop::AST::BuilderPrism
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def create_parser(ruby_version, parser_engine, prism_result)
+        builder = builder_class(parser_engine).new
+
+        parser_class = parser_class(ruby_version, parser_engine)
+
+        parser_instance = if parser_engine == :parser_prism && prism_result
+                            # NOTE: Since it is intended for use with Ruby LSP, it targets only Prism.
+                            # If there is no reuse of a pre-parsed result, such as in Ruby LSP,
+                            # regular parsing with Prism occurs, and `else` branch will be executed.
+                            prism_reparsed = PrismPreparsed.new(prism_result)
+                            parser_class.new(builder, parser: prism_reparsed)
+                          else
+                            parser_class.new(builder)
+                          end
 
-        parser_class(ruby_version).new(builder).tap do |parser|
+        parser_instance.tap do |parser|
           # On JRuby there's a risk that we hang in tokenize() if we
           # don't set the all errors as fatal flag. The problem is caused by a bug
           # in Racc that is discussed in issue #93 of the whitequark/parser
@@ -287,6 +363,32 @@ module RuboCop
           end
         end
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      def normalize_parser_engine(parser_engine, ruby_version)
+        parser_engine = parser_engine.to_sym
+        unless PARSER_ENGINES.include?(parser_engine)
+          raise ArgumentError, 'The keyword argument `parser_engine` accepts `default`, ' \
+                               "`parser_whitequark`, or `parser_prism`, but `#{parser_engine}` " \
+                               'was passed.'
+        end
+        if parser_engine == :default
+          default_parser_engine(ruby_version)
+        else
+          parser_engine
+        end
+      end
+
+      # The Parser gem does not support Ruby 3.5 or later.
+      # It is also not fully compatible with Ruby 3.4 but for
+      # now respects using parser for backwards compatibility.
+      def default_parser_engine(ruby_version)
+        if ruby_version >= 3.4
+          :parser_prism
+        else
+          :parser_whitequark
+        end
+      end
 
       def first_token_index(range_or_node)
         begin_pos = source_range(range_or_node).begin_pos
@@ -308,4 +410,3 @@ module RuboCop
     end
   end
 end
-# rubocop:enable Metrics/ClassLength

data/lib/rubocop/ast/token.rb

@@ -5,6 +5,7 @@ module RuboCop
     # A basic wrapper around Parser's tokens.
     class Token
       LEFT_PAREN_TYPES = %i[tLPAREN tLPAREN2].freeze
+      LEFT_CURLY_TYPES = %i[tLCURLY tLAMBEG].freeze
 
       attr_reader :pos, :type, :text
 
@@ -83,7 +84,7 @@ module RuboCop
       end
 
       def left_curly_brace?
-        type == :tLCURLY
+        LEFT_CURLY_TYPES.include?(type)
       end
 
       def right_curly_brace?

data/lib/rubocop/ast/traversal.rb

@@ -26,39 +26,48 @@ module RuboCop
         SEND = 'send(TYPE_TO_METHOD[child.type], child)'
         assign_code = 'child = node.children[%<index>i]'
         code = "#{assign_code}\n#{SEND}"
+        # How a particular child node should be visited. For example, if a child node
+        # can be nil it should be guarded behind a nil check. Or, if a child node is a literal
+        # (like a symbol) then the literal itself should not be visited.
         TEMPLATE = {
           skip: '',
           always: code,
           nil?: "#{code} if child"
         }.freeze
 
-        def def_callback(type, *signature,
-                         arity: signature.size..signature.size,
-                         arity_check: ENV.fetch('RUBOCOP_DEBUG', nil) && self.arity_check(arity),
-                         body: self.body(signature, arity_check))
+        def def_callback(type, *child_node_types,
+                         expected_children_count: child_node_types.size..child_node_types.size,
+                         body: self.body(child_node_types, expected_children_count))
           type, *aliases = type
           lineno = caller_locations(1, 1).first.lineno
-          module_eval(<<~RUBY, __FILE__, lineno) # rubocop:disable Style/EvalWithLocation
+          module_eval(<<~RUBY, __FILE__, lineno)
             def on_#{type}(node)        # def on_send(node)
               #{body}                   #   # body ...
               nil                       #   nil
             end                         # end
           RUBY
           aliases.each do |m|
-            alias_method "on_#{m}", "on_#{type}"
+            alias_method :"on_#{m}", :"on_#{type}"
           end
         end
 
-        def body(signature, prelude)
-          signature
-            .map.with_index do |arg, i|
-              TEMPLATE[arg].gsub('%<index>i', i.to_s)
+        def body(child_node_types, expected_children_count)
+          visit_children_code =
+            child_node_types
+            .map.with_index do |child_type, i|
+              TEMPLATE.fetch(child_type).gsub('%<index>i', i.to_s)
             end
-            .unshift(prelude)
             .join("\n")
+
+          <<~BODY
+            #{children_count_check_code(expected_children_count)}
+            #{visit_children_code}
+          BODY
         end
 
-        def arity_check(range)
+        def children_count_check_code(range)
+          return '' unless ENV.fetch('RUBOCOP_DEBUG', false)
+
           <<~RUBY
             n = node.children.size
             raise DebugError, [
@@ -72,18 +81,18 @@ module RuboCop
       extend CallbackCompiler
       send_code = CallbackCompiler::SEND
 
-      ### arity == 0
+      ### children count == 0
       no_children = %i[true false nil self cbase zsuper redo retry
                        forward_args forwarded_args match_nil_pattern
                        forward_arg forwarded_restarg forwarded_kwrestarg
                        lambda empty_else kwnilarg
                        __FILE__ __LINE__ __ENCODING__]
 
-      ### arity == 0..1
+      ### children count == 0..1
       opt_symbol_child = %i[restarg kwrestarg]
       opt_node_child = %i[splat kwsplat match_rest]
 
-      ### arity == 1
+      ### children count == 1
       literal_child = %i[int float complex
                          rational str sym lvar
                          ivar cvar gvar nth_ref back_ref
@@ -100,16 +109,16 @@ module RuboCop
       NO_CHILD_NODES = (no_children + opt_symbol_child + literal_child).to_set.freeze
       private_constant :NO_CHILD_NODES # Used by Commissioner
 
-      ### arity > 1
+      ### children count > 1
       symbol_then_opt_node = %i[lvasgn ivasgn cvasgn gvasgn]
       symbol_then_node_or_nil = %i[optarg kwoptarg]
       node_then_opt_node = %i[while until module sclass]
 
-      ### variable arity
+      ### variable children count
       many_node_children = %i[dstr dsym xstr regexp array hash pair
                               mlhs masgn or_asgn and_asgn rasgn mrasgn
                               undef alias args super yield or and
-                              while_post until_post iflipflop eflipflop
+                              while_post until_post
                               match_with_lvasgn begin kwbegin return
                               in_match match_alt break next
                               match_as array_pattern array_pattern_with_tail
@@ -117,22 +126,22 @@ module RuboCop
                               index indexasgn procarg0 kwargs]
       many_opt_node_children = %i[case rescue resbody ensure for when
                                   case_match in_pattern irange erange
-                                  match_pattern match_pattern_p]
+                                  match_pattern match_pattern_p iflipflop eflipflop]
 
       ### Callbacks for above
       def_callback no_children
-      def_callback opt_symbol_child, :skip, arity: 0..1
-      def_callback opt_node_child, :nil?, arity: 0..1
+      def_callback opt_symbol_child, :skip, expected_children_count: 0..1
+      def_callback opt_node_child, :nil?, expected_children_count: 0..1
 
       def_callback literal_child, :skip
       def_callback node_child, :always
       def_callback node_or_nil_child, :nil?
 
-      def_callback symbol_then_opt_node, :skip, :nil?, arity: 1..2
+      def_callback symbol_then_opt_node, :skip, :nil?, expected_children_count: 1..2
       def_callback symbol_then_node_or_nil, :skip, :nil?
       def_callback node_then_opt_node, :always, :nil?
 
-      def_callback many_symbol_children, :skip, arity_check: nil
+      def_callback many_symbol_children, :skip, expected_children_count: (0..)
       def_callback many_node_children, body: <<~RUBY
         node.children.each { |child| #{send_code} }
       RUBY
@@ -143,13 +152,14 @@ module RuboCop
 
       ### Other particular cases
       def_callback :const, :nil?, :skip
-      def_callback :casgn, :nil?, :skip, :nil?, arity: 2..3
+      def_callback :casgn, :nil?, :skip, :nil?, expected_children_count: 2..3
       def_callback :class, :always, :nil?, :nil?
       def_callback :def, :skip, :always, :nil?
       def_callback :op_asgn, :always, :skip, :always
       def_callback :if, :always, :nil?, :nil?
       def_callback :block, :always, :always, :nil?
       def_callback :numblock, :always, :skip, :nil?
+      def_callback :itblock, :always, :skip, :nil?
       def_callback :defs, :always, :skip, :always, :nil?
 
       def_callback %i[send csend], body: <<~RUBY
@@ -167,7 +177,7 @@ module RuboCop
 
       to_define = ::Parser::Meta::NODE_TYPES.to_a
       to_define -= defined
-      to_define -= %i[numargs ident] # transient
+      to_define -= %i[numargs itarg ident] # transient
       to_define -= %i[blockarg_expr restarg_expr] # obsolete
       to_define -= %i[objc_kwarg objc_restarg objc_varargs] # mac_ruby
       def_callback to_define, body: <<~RUBY

data/lib/rubocop/ast/utilities/simple_forwardable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module RuboCop
+  # Similar to `Forwardable#def_delegators`, but simpler & faster
+  module SimpleForwardable
+    def def_delegators(accessor, *methods)
+      methods.each do |method|
+        if method.end_with?('=') && method.to_s != '[]='
+          # Defining a delegator for `foo=` can't use `foo=(...)` because it is a
+          # syntax error. Fall back to doing a slower `public_send` instead.
+          # TODO: Use foo(method, ...) when Ruby 3.1 is required.
+          class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
+            def #{method}(*args, **kwargs, &blk)                         # def example=(*args, **kwargs, &blk)
+              #{accessor}.public_send(:#{method}, *args, **kwargs, &blk) #   foo.public_send(:example=, *args, **kwargs, &blk)
+            end                                                          # end
+          RUBY
+        else
+          class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
+            def #{method}(...)           # def example(...)
+              #{accessor}.#{method}(...) #   foo.example(...)
+            end                          # end
+          RUBY
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/ast/version.rb

@@ -3,7 +3,7 @@
 module RuboCop
   module AST
     module Version
-      STRING = '1.24.1'
+      STRING = '1.46.0'
     end
   end
 end

data/lib/rubocop/ast.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
 require 'parser'
+require 'prism'
 require 'forwardable'
 require 'set'
 
 require_relative 'ast/ext/range'
-require_relative 'ast/ext/range_min_max'
+require_relative 'ast/utilities/simple_forwardable'
 require_relative 'ast/node_pattern/method_definer'
 require_relative 'ast/node_pattern'
 require_relative 'ast/node/mixin/descendence'
@@ -27,6 +28,7 @@ require_relative 'ast/node/mixin/method_identifier_predicates'
 require_relative 'ast/node/mixin/binary_operator_node'
 require_relative 'ast/node/mixin/collection_node'
 require_relative 'ast/node/mixin/conditional_node'
+require_relative 'ast/node/mixin/constant_node'
 require_relative 'ast/node/mixin/hash_element_node'
 require_relative 'ast/node/mixin/method_dispatch_node'
 require_relative 'ast/node/mixin/modifier_node'
@@ -46,6 +48,7 @@ require_relative 'ast/node/case_match_node'
 require_relative 'ast/node/case_node'
 require_relative 'ast/node/casgn_node'
 require_relative 'ast/node/class_node'
+require_relative 'ast/node/complex_node'
 require_relative 'ast/node/const_node'
 require_relative 'ast/node/def_node'
 require_relative 'ast/node/defined_node'
@@ -59,8 +62,11 @@ require_relative 'ast/node/in_pattern_node'
 require_relative 'ast/node/index_node'
 require_relative 'ast/node/indexasgn_node'
 require_relative 'ast/node/int_node'
+require_relative 'ast/node/keyword_begin_node'
 require_relative 'ast/node/keyword_splat_node'
 require_relative 'ast/node/lambda_node'
+require_relative 'ast/node/masgn_node'
+require_relative 'ast/node/mlhs_node'
 require_relative 'ast/node/module_node'
 require_relative 'ast/node/next_node'
 require_relative 'ast/node/op_asgn_node'
@@ -70,6 +76,7 @@ require_relative 'ast/node/or_node'
 require_relative 'ast/node/pair_node'
 require_relative 'ast/node/procarg0_node'
 require_relative 'ast/node/range_node'
+require_relative 'ast/node/rational_node'
 require_relative 'ast/node/regexp_node'
 require_relative 'ast/node/rescue_node'
 require_relative 'ast/node/resbody_node'
@@ -82,10 +89,12 @@ require_relative 'ast/node/dstr_node'
 require_relative 'ast/node/super_node'
 require_relative 'ast/node/symbol_node'
 require_relative 'ast/node/until_node'
+require_relative 'ast/node/var_node'
 require_relative 'ast/node/when_node'
 require_relative 'ast/node/while_node'
 require_relative 'ast/node/yield_node'
 require_relative 'ast/builder'
+require_relative 'ast/builder_prism'
 require_relative 'ast/processed_source'
 require_relative 'ast/rubocop_compatibility'
 require_relative 'ast/token'