rubocop-ast 1.49.1 → 1.50.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aad4bf15828f1db50f117c1a983e1f4c7f0cff233878a7f1a1686416ab21feff
-  data.tar.gz: 8ac8ad74b061c5fcd733e21420068edc69d0a9bfba532dc766e01dbb80be5c40
+  metadata.gz: 21fa08d16f9a915d62568fbf85b25efeefbf23fba1bf710b772130bcdcced47d
+  data.tar.gz: 2789ea768313b62f16dc6a6f77e3f4b02e5d1af01a7d765483284fb3cd65c44c
 SHA512:
-  metadata.gz: 75f3882c819f2e10f7ef893481fbe9feddedfcdc91678c1c695d6ef36d94798dedf5667427cbfc6c022494e7e0c9581f54b4e1dcc9f05f2aea677668fe645b04
-  data.tar.gz: 94b37b0490cbf1ab81ab0c4c98a5bd8e6037b6ae5210af6ba2cbd23d4a263a3e4cfccad3f99fe621a6ab62d8356fc007fd066adb2d2379e02f5b07580ef2b2d4
+  metadata.gz: 53cd0e762e23e4394cbd01ed3d2ae6a86983e22f0a18d7782207874d691140a6eca839e67f2ee746ffb942863c65c3278cde553b408cf8c1fdfb0da736e393e4
+  data.tar.gz: d8657ab6d0d89f0d9d963e13379283c7aba9245fb9fa3227316f4b7955d6faab37a2da0539814461c5b39e5d8e5c7c12d715c8ebcdd2406e606e7e6f350e0160

data/lib/rubocop/ast/node/and_node.rb

@@ -2,9 +2,9 @@
 
 module RuboCop
   module AST
-    # A node extension for `until` nodes. This will be used in place of a plain
+    # A node extension for `and` nodes. This will be used in place of a plain
     # node when the builder constructs the AST, making its methods available
-    # to all `until` nodes within RuboCop.
+    # to all `and` nodes within RuboCop.
     class AndNode < Node
       include BinaryOperatorNode
       include PredicateOperatorNode

data/lib/rubocop/ast/node/case_match_node.rb

@@ -17,7 +17,7 @@ module RuboCop
 
       # @deprecated Use `in_pattern_branches.each`
       def each_in_pattern(&block)
-        return in_pattern_branches.to_enum(__method__) unless block
+        return to_enum(__method__) unless block
 
         in_pattern_branches.each(&block)
 

data/lib/rubocop/ast/node/case_node.rb

@@ -17,7 +17,7 @@ module RuboCop
 
       # @deprecated Use `when_branches.each`
       def each_when(&block)
-        return when_branches.to_enum(__method__) unless block
+        return to_enum(__method__) unless block
 
         when_branches.each(&block)
 

data/lib/rubocop/ast/node/for_node.rb

@@ -8,7 +8,7 @@ module RuboCop
     class ForNode < Node
       # Returns the keyword of the `for` statement as a string.
       #
-      # @return [String] the keyword of the `until` statement
+      # @return [String] the keyword of the `for` statement
       def keyword
         'for'
       end

data/lib/rubocop/ast/node/hash_node.rb

@@ -56,10 +56,10 @@ module RuboCop
       #
       # @return [self] if a block is given
       # @return [Enumerator] if no block is given
-      def each_key(&block)
-        return pairs.map(&:key).to_enum unless block
+      def each_key
+        return to_enum(__method__) unless block_given?
 
-        pairs.map(&:key).each(&block)
+        each_child_node(:pair) { |pair| yield pair.key }
 
         self
       end
@@ -80,10 +80,10 @@ module RuboCop
       #
       # @return [self] if a block is given
       # @return [Enumerator] if no block is given
-      def each_value(&block)
-        return pairs.map(&:value).to_enum unless block
+      def each_value
+        return to_enum(__method__) unless block_given?
 
-        pairs.map(&:value).each(&block)
+        each_child_node(:pair) { |pair| yield pair.value }
 
         self
       end

data/lib/rubocop/ast/node/if_node.rb

@@ -169,7 +169,7 @@ module RuboCop
 
       # @deprecated Use `branches.each`
       def each_branch(&block)
-        return branches.to_enum(__method__) unless block
+        return to_enum(__method__) unless block
 
         branches.each(&block)
       end

data/lib/rubocop/ast/node/mixin/descendence.rb

@@ -25,7 +25,7 @@ module RuboCop
         children.each do |child|
           next unless child.is_a?(::AST::Node)
 
-          yield child if types.empty? || child.type?(*types)
+          yield child if types.empty? || child.type_in?(types)
         end
 
         self
@@ -95,7 +95,7 @@ module RuboCop
       def each_node(*types, &block)
         return to_enum(__method__, *types) unless block
 
-        yield self if types.empty? || type?(*types)
+        yield self if types.empty? || type_in?(types)
 
         visit_descendants(types, &block)
 
@@ -105,11 +105,28 @@ module RuboCop
       protected
 
       def visit_descendants(types, &block)
+        if types.empty?
+          visit_all_descendants(&block)
+        else
+          visit_descendants_of_types(types, &block)
+        end
+      end
+
+      def visit_all_descendants(&block)
+        children.each do |child|
+          next unless child.is_a?(::AST::Node)
+
+          yield child
+          child.visit_all_descendants(&block)
+        end
+      end
+
+      def visit_descendants_of_types(types, &block)
         children.each do |child|
           next unless child.is_a?(::AST::Node)
 
-          yield child if types.empty? || child.type?(*types)
-          child.visit_descendants(types, &block)
+          yield child if child.type_in?(types)
+          child.visit_descendants_of_types(types, &block)
         end
       end
     end

data/lib/rubocop/ast/node/mixin/method_dispatch_node.rb

@@ -165,7 +165,7 @@ module RuboCop
       #
       # @return [Boolean] whether the dispatched method has a block
       def block_literal?
-        parent&.any_block_type? && eql?(parent.send_node)
+        parent&.any_block_type? && equal?(parent.send_node)
       end
 
       # Checks whether this node is an arithmetic operation

data/lib/rubocop/ast/node/mixin/method_identifier_predicates.rb

@@ -140,7 +140,7 @@ module RuboCop
       #
       # @return [Boolean] whether the method is an assignment
       def assignment_method?
-        !comparison_method? && method_name.to_s.end_with?('=')
+        !comparison_method? && method_name.end_with?('=')
       end
 
       # Checks whether the method is an enumerator method.
@@ -148,7 +148,7 @@ module RuboCop
       # @return [Boolean] whether the method is an enumerator
       def enumerator_method?
         ENUMERATOR_METHODS.include?(method_name) ||
-          method_name.to_s.start_with?('each_')
+          method_name.start_with?('each_')
       end
 
       # Checks whether the method is an Enumerable method.
@@ -162,14 +162,14 @@ module RuboCop
       #
       # @return [Boolean] whether the method is a predicate method
       def predicate_method?
-        method_name.to_s.end_with?('?')
+        method_name.end_with?('?')
       end
 
       # Checks whether the method is a bang method.
       #
       # @return [Boolean] whether the method is a bang method
       def bang_method?
-        method_name.to_s.end_with?('!')
+        method_name.end_with?('!')
       end
 
       # Checks whether the method is a camel case method,
@@ -177,7 +177,7 @@ module RuboCop
       #
       # @return [Boolean] whether the method is a camel case method
       def camel_case_method?
-        method_name.to_s =~ /\A[A-Z]/
+        method_name.match?(/\A[A-Z]/)
       end
 
       # Checks whether the *explicit* receiver of this node is `self`.

data/lib/rubocop/ast/node/mixin/numeric_node.rb

@@ -7,7 +7,7 @@ module RuboCop
       SIGN_REGEX = /\A[+-]/.freeze
       private_constant :SIGN_REGEX
 
-      # Checks whether this is literal has a sign.
+      # Checks whether this literal has a sign.
       #
       # @example
       #

data/lib/rubocop/ast/node/pair_node.rb

@@ -32,7 +32,7 @@ module RuboCop
       end
 
       # Returns the delimiter of the `pair` as a string. Returns `=>` for a
-      # colon delimited `pair` and `:` for a hash rocket delimited `pair`.
+      # hash rocket delimited `pair` and `:` for a colon delimited `pair`.
       #
       # @param [Boolean] with_spacing whether to include spacing
       # @return [String] the delimiter of the `pair`

data/lib/rubocop/ast/node/symbol_node.rb

@@ -2,7 +2,7 @@
 
 module RuboCop
   module AST
-    # A node extension for `sym` nodes. This will be used in  place of a
+    # A node extension for `sym` nodes. This will be used in place of a
     # plain node when the builder constructs the AST, making its methods
     # available to all `sym` nodes within RuboCop.
     class SymbolNode < Node

data/lib/rubocop/ast/node/when_node.rb

@@ -15,7 +15,7 @@ module RuboCop
 
       # @deprecated Use `conditions.each`
       def each_condition(&block)
-        return conditions.to_enum(__method__) unless block
+        return to_enum(__method__) unless block
 
         conditions.each(&block)
 

data/lib/rubocop/ast/node/while_node.rb

@@ -24,9 +24,9 @@ module RuboCop
         'until'
       end
 
-      # Checks whether the `until` node has a `do` keyword.
+      # Checks whether the `while` node has a `do` keyword.
       #
-      # @return [Boolean] whether the `until` node has a `do` keyword
+      # @return [Boolean] whether the `while` node has a `do` keyword
       def do?
         loc_is?(:begin, 'do')
       end

data/lib/rubocop/ast/node.rb

@@ -76,6 +76,13 @@ module RuboCop
       OPERATOR_KEYWORDS = %i[and or].to_set.freeze
       # @api private
       SPECIAL_KEYWORDS = %w[__FILE__ __LINE__ __ENCODING__].to_set.freeze
+      # The node types which could have a source matching `SPECIAL_KEYWORDS`:
+      # `__FILE__` parses as `str`, `__LINE__` as `int` and `__ENCODING__` as
+      # `const` (or as their own node types when the builder emits them);
+      # `str` and `sym` also cover word/symbol array elements, string parts
+      # and hash labels spelling out one of these keywords.
+      SPECIAL_KEYWORD_TYPES = %i[str sym int const __FILE__ __LINE__ __ENCODING__].to_set.freeze
+      private_constant :SPECIAL_KEYWORD_TYPES
 
       LITERAL_RECURSIVE_METHODS = (COMPARISON_OPERATORS + %i[* ! <=>]).freeze
       LITERAL_RECURSIVE_TYPES = (OPERATOR_KEYWORDS + COMPOSITE_LITERALS + %i[begin pair]).freeze
@@ -172,6 +179,13 @@ module RuboCop
       # Allows specific single node types, as well as "grouped" types
       # (e.g. `:boolean` for `:true` or `:false`)
       def type?(*types)
+        type_in?(types)
+      end
+
+      # Non-splatting variant of `type?`, used by the traversal hot paths to
+      # avoid allocating an array per visited node.
+      # @api private
+      def type_in?(types)
         return true if types.include?(type)
 
         group_type = GROUP_FOR_TYPE[type]
@@ -287,7 +301,7 @@ module RuboCop
       # Some node types override this with their own custom
       # destructuring method.
       #
-      # @return [Array<Node>] the different parts of the ndde
+      # @return [Array<Node>] the different parts of the node
       alias node_parts to_a
 
       # Calls the given block for each ancestor node from parent to root.
@@ -503,7 +517,7 @@ module RuboCop
       end
 
       def special_keyword?
-        SPECIAL_KEYWORDS.include?(source)
+        SPECIAL_KEYWORD_TYPES.include?(type) && SPECIAL_KEYWORDS.include?(source)
       end
 
       def operator_keyword?
@@ -519,11 +533,13 @@ module RuboCop
       end
 
       def chained?
-        parent&.call_type? && eql?(parent.receiver)
+        parent&.call_type? && equal?(parent.receiver)
       end
 
       def argument?
-        parent&.send_type? && parent.arguments.include?(self)
+        return false unless parent&.send_type?
+
+        parent.arguments.any? { |argument| argument.equal?(self) }
       end
 
       def any_def_type?
@@ -695,9 +711,16 @@ module RuboCop
       def visit_ancestors(types)
         last_node = self
 
-        while (current_node = last_node.parent)
-          yield current_node if types.empty? || current_node.type?(*types)
-          last_node = current_node
+        if types.empty?
+          while (current_node = last_node.parent)
+            yield current_node
+            last_node = current_node
+          end
+        else
+          while (current_node = last_node.parent)
+            yield current_node if current_node.type_in?(types)
+            last_node = current_node
+          end
         end
       end
 

data/lib/rubocop/ast/processed_source.rb

@@ -25,6 +25,44 @@ module RuboCop
       end
     end
 
+    # Extends the prism translation parsers so that the conversion of tokens
+    # into the `parser` gem's format is deferred until the tokens are first
+    # accessed. Building the tokens is a significant part of the translation
+    # cost, and not every caller needs them.
+    # @api private
+    module PrismLazyTokens
+      # Same contract as `Parser::Base#tokenize`, except the tokens are
+      # returned as a callable that performs the conversion when invoked.
+      def tokenize_deferred(source_buffer)
+        @source_buffer = source_buffer
+        source = source_buffer.source
+
+        offset_cache = build_offset_cache(source)
+        result = unwrap(@parser.parse_lex(source, **prism_options), offset_cache)
+
+        program, tokens = result.value
+        ast = build_ast(program, offset_cache) if result.success?
+        comments = build_comments(result.comments, offset_cache)
+
+        [ast, comments, deferred_tokens(source_buffer, tokens, offset_cache)]
+      ensure
+        @source_buffer = nil
+      end
+
+      private
+
+      def deferred_tokens(source_buffer, tokens, offset_cache)
+        lambda do
+          @source_buffer = source_buffer
+          begin
+            build_tokens(tokens, offset_cache)
+          ensure
+            @source_buffer = nil
+          end
+        end
+      end
+    end
+
     # ProcessedSource contains objects which are generated by Parser
     # and other information such as disabled lines for cops.
     # It also provides a convenient way to access source lines.
@@ -38,7 +76,7 @@ module RuboCop
       PARSER_ENGINES = %i[default parser_whitequark parser_prism].freeze
       private_constant :PARSER_ENGINES
 
-      attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
+      attr_reader :path, :buffer, :ast, :comments, :diagnostics,
                   :parser_error, :raw_source, :ruby_version, :parser_engine
 
       def self.from_file(path, ruby_version, parser_engine: :default)
@@ -46,6 +84,13 @@ module RuboCop
         new(file, ruby_version, path, parser_engine: parser_engine)
       end
 
+      # Subclasses of the prism translation parsers with lazily built tokens.
+      # @api private
+      def self.lazy_tokens_parser_class(base)
+        @lazy_tokens_parser_classes ||= {}
+        @lazy_tokens_parser_classes[base] ||= Class.new(base) { include PrismLazyTokens }
+      end
+
       def initialize(
         source, ruby_version, path = nil, parser_engine: :default, prism_result: nil
       )
@@ -54,7 +99,7 @@ module RuboCop
         # Defaults source encoding to UTF-8, regardless of the encoding it has
         # been read with, which could be non-utf8 depending on the default
         # external encoding.
-        (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8
+        source = (+source).force_encoding(Encoding::UTF_8) unless source.encoding == Encoding::UTF_8
 
         @raw_source = source
         @path = path
@@ -77,14 +122,13 @@ module RuboCop
       def lines
         @lines ||= begin
           all_lines = @buffer.source_lines
-          last_token_line = tokens.any? ? tokens.last.line : all_lines.size
-          result = []
-          all_lines.each_with_index do |line, ix|
-            break if ix >= last_token_line && line == '__END__'
-
-            result << line
+          if all_lines.include?('__END__')
+            trim_lines_after_data_marker(all_lines)
+          else
+            # Don't consult the tokens (which may have to be built first) when
+            # the source can't contain a data section.
+            all_lines.dup
           end
-          result
         end
       end
 
@@ -100,7 +144,7 @@ module RuboCop
 
       # Raw source checksum for tracking infinite loops.
       def checksum
-        Digest::SHA1.hexdigest(@raw_source)
+        @checksum ||= Digest::SHA1.hexdigest(@raw_source)
       end
 
       # @deprecated Use `comments.each`
@@ -173,6 +217,8 @@ module RuboCop
       end
 
       def preceding_line(token)
+        return nil if token.line < 2
+
         lines[token.line - 2]
       end
 
@@ -191,6 +237,19 @@ module RuboCop
           .length
       end
 
+      # The tokens of the source. With the prism engine the tokens are built
+      # lazily on first access, since their conversion is costly and not
+      # every caller needs them.
+      def tokens
+        @tokens ||= begin
+          tokens = parser_tokens.map { |t| Token.from_parser_token(t) }
+          # The parser tokens are no longer needed, and the deferred conversion
+          # holds on to the parser instance, so release both.
+          @parser_tokens = nil
+          tokens
+        end
+      end
+
       def tokens_within(range_or_node)
         begin_index = first_token_index(range_or_node)
         end_index = last_token_index(range_or_node)
@@ -209,12 +268,36 @@ module RuboCop
       # is passed as a method argument. In this case tokens are interleaved by
       # heredoc contents' tokens.
       def sorted_tokens
-        # Use stable sort.
-        @sorted_tokens ||= tokens.sort_by.with_index { |token, i| [token.begin_pos, i] }
+        # Most sources have their tokens already in order, in which case
+        # sorting can be skipped entirely. Callers only ever read from the
+        # returned array, so it is safe to reuse `tokens` as is.
+        @sorted_tokens ||= if tokens_sorted?
+                             tokens
+                           else
+                             # Use stable sort.
+                             tokens.sort_by.with_index { |token, i| [token.begin_pos, i] }
+                           end
       end
 
       private
 
+      def tokens_sorted?
+        tokens.each_cons(2).all? { |a, b| a.begin_pos <= b.begin_pos }
+      end
+
+      # `__END__` only starts a data section when it isn't nested inside a
+      # string or heredoc, which is what the last token's line disambiguates.
+      def trim_lines_after_data_marker(all_lines)
+        last_token_line = tokens.any? ? tokens.last.line : all_lines.size
+        result = []
+        all_lines.each_with_index do |line, ix|
+          break if ix >= last_token_line && line == '__END__'
+
+          result << line
+        end
+        result
+      end
+
       def comment_index
         @comment_index ||= {}.tap do |hash|
           comments.each { |c| hash[c.location.line] = c }
@@ -222,8 +305,7 @@ module RuboCop
       end
 
       def parse(source, ruby_version, parser_engine, prism_result)
-        buffer_name = @path || STRING_SOURCE_NAME
-        @buffer = Parser::Source::Buffer.new(buffer_name, 1)
+        @buffer = Parser::Source::Buffer.new(@path || STRING_SOURCE_NAME, 1)
 
         begin
           @buffer.source = source
@@ -237,12 +319,23 @@ module RuboCop
 
         parser = create_parser(ruby_version, parser_engine, prism_result)
 
-        @ast, @comments, @tokens = tokenize(parser)
+        @ast, @comments, tokens = tokenize(parser)
+        store_tokens(tokens)
+      end
+
+      # The tokens may be an already converted array, or a deferred conversion
+      # to be performed when the tokens are first accessed.
+      def store_tokens(tokens)
+        if tokens.is_a?(Proc)
+          @deferred_parser_tokens = tokens
+        else
+          @parser_tokens = tokens
+        end
       end
 
       def tokenize(parser)
         begin
-          ast, comments, tokens = parser.tokenize(@buffer)
+          ast, comments, tokens = parse_and_lex(parser)
           ast ||= nil # force `false` to `nil`, see https://github.com/whitequark/parser/pull/722
         rescue Parser::SyntaxError
           # All errors are in diagnostics. No need to handle exception.
@@ -251,11 +344,27 @@ module RuboCop
         end
 
         ast&.complete!
-        tokens.map! { |t| Token.from_parser_token(t) }
 
         [ast, comments, tokens]
       end
 
+      def parse_and_lex(parser)
+        if parser.respond_to?(:tokenize_deferred)
+          parser.tokenize_deferred(@buffer)
+        else
+          parser.tokenize(@buffer)
+        end
+      end
+
+      def parser_tokens
+        if @deferred_parser_tokens
+          @parser_tokens = @deferred_parser_tokens.call
+          @deferred_parser_tokens = nil
+        end
+
+        @parser_tokens
+      end
+
       # rubocop:disable Lint/FloatComparison, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
       def parser_class(ruby_version, parser_engine)
         case parser_engine
@@ -340,6 +449,9 @@ module RuboCop
 
         parser_class = parser_class(ruby_version, parser_engine)
 
+        parser_class = self.class.lazy_tokens_parser_class(parser_class) if
+          parser_engine == :parser_prism
+
         parser_instance = if parser_engine == :parser_prism && prism_result
                             # NOTE: Since it is intended for use with Ruby LSP, it targets only Prism.
                             # If there is no reuse of a pre-parsed result, such as in Ruby LSP,
@@ -378,11 +490,11 @@ module RuboCop
         end
       end
 
-      # The Parser gem does not support Ruby 3.5 or later.
-      # It is also not fully compatible with Ruby 3.4 but for
-      # now respects using parser for backwards compatibility.
+      # Prism is used for all Ruby versions it can parse (3.3 and later);
+      # the Parser gem does not support Ruby 3.5 or later and is not fully
+      # compatible with Ruby 3.4.
       def default_parser_engine(ruby_version)
-        if ruby_version >= 3.4
+        if ruby_version >= 3.3
           :parser_prism
         else
           :parser_whitequark

data/lib/rubocop/ast/version.rb

@@ -3,7 +3,7 @@
 module RuboCop
   module AST
     module Version
-      STRING = '1.49.1'
+      STRING = '1.50.0'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubocop-ast
 version: !ruby/object:Gem::Version
-  version: 1.49.1
+  version: 1.50.0
 platform: ruby
 authors:
 - Bozhidar Batsov