rubocop 1.79.2 → 1.88.0

data/lib/rubocop/result_cache.rb

@@ -9,7 +9,7 @@ module RuboCop
   # Provides functionality for caching RuboCop runs.
   # @api private
   class ResultCache
-    NON_CHANGING = %i[color format formatters out debug fail_level
+    NON_CHANGING = %i[color format formatters out debug display_time fail_level
                       fix_layout autocorrect safe_autocorrect autocorrect_all
                       cache fail_fast stdin parallel].freeze
 
@@ -22,19 +22,22 @@ module RuboCop
     # Remove old files so that the cache doesn't grow too big. When the
     # threshold MaxFilesInCache has been exceeded, the oldest 50% of all the
     # files in the cache are removed. The reason for removing so much is that
-    # cleaning should be done relatively seldom, since there is a slight risk
+    # removing should be done relatively seldom, since there is a slight risk
     # that some other RuboCop process was just about to read the file, when
     # there's parallel execution and the cache is shared.
     def self.cleanup(config_store, verbose, cache_root_override = nil)
       return if inhibit_cleanup # OPTIMIZE: For faster testing
+      return unless config_store.for_pwd.for_all_cops['MaxFilesInCache']
 
       rubocop_cache_dir = cache_root(config_store, cache_root_override)
       return unless File.exist?(rubocop_cache_dir)
 
-      files, dirs = Find.find(rubocop_cache_dir).partition { |path| File.file?(path) }
+      # We know the cache entries are 3 level deep, so globing
+      # for `*/*/*` only returns files.
+      files = Dir[File.join(rubocop_cache_dir, '*/*/*')]
       return unless requires_file_removal?(files.length, config_store)
 
-      remove_oldest_files(files, dirs, rubocop_cache_dir, verbose)
+      remove_oldest_files(files, rubocop_cache_dir, verbose)
     end
 
     class << self
@@ -49,39 +52,57 @@ module RuboCop
         file_count > 1 && file_count > config_store.for_pwd.for_all_cops['MaxFilesInCache']
       end
 
-      def remove_oldest_files(files, dirs, rubocop_cache_dir, verbose)
+      def remove_oldest_files(files, rubocop_cache_dir, verbose)
         # Add 1 to half the number of files, so that we remove the file if
         # there's only 1 left.
         remove_count = (files.length / 2) + 1
         puts "Removing the #{remove_count} oldest files from #{rubocop_cache_dir}" if verbose
         sorted = files.sort_by { |path| File.mtime(path) }
-        remove_files(sorted, dirs, remove_count)
+        remove_files(sorted, remove_count)
       rescue Errno::ENOENT
         # This can happen if parallel RuboCop invocations try to remove the
         # same files. No problem.
         puts $ERROR_INFO if verbose
       end
 
-      def remove_files(files, dirs, remove_count)
+      def remove_files(files, remove_count)
         # Batch file deletions, deleting over 130,000+ files will crash
         # File.delete.
         files[0, remove_count].each_slice(10_000).each do |files_slice|
           File.delete(*files_slice)
         end
-        dirs.each { |dir| Dir.rmdir(dir) if Dir["#{dir}/*"].empty? }
+
+        dirs = files.map { |f| File.dirname(f) }.uniq
+        until dirs.empty?
+          dirs.select! do |dir|
+            Dir.rmdir(dir)
+            true
+          rescue SystemCallError # ENOTEMPTY etc
+            false
+          end
+          dirs = dirs.map { |f| File.dirname(f) }.uniq
+        end
       end
     end
 
     def self.cache_root(config_store, cache_root_override = nil)
-      CacheConfig.root_dir do
+      return @cache_root if @cache_root && !cache_root_override
+
+      result = CacheConfig.root_dir do
         cache_root_override || config_store.for_pwd.for_all_cops['CacheRootDirectory']
       end
+      @cache_root = result unless cache_root_override
+      result
     end
 
     def self.allow_symlinks_in_cache_location?(config_store)
       config_store.for_pwd.for_all_cops['AllowSymlinksInCacheRootDirectory']
     end
 
+    def self.reset_config_cache
+      @cache_root = nil
+    end
+
     attr_reader :path
 
     def initialize(file, team, options, config_store, cache_root_override = nil)
@@ -90,7 +111,7 @@ module RuboCop
       @allow_symlinks_in_cache_location =
         ResultCache.allow_symlinks_in_cache_location?(config_store)
       @path = File.join(rubocop_cache_dir,
-                        rubocop_checksum,
+                        self.class.source_checksum,
                         context_checksum(team, options),
                         file_checksum(file, config_store))
       @cached_data = CachedData.new(file)
@@ -167,13 +188,11 @@ module RuboCop
     end
 
     class << self
-      attr_accessor :source_checksum, :inhibit_cleanup
-    end
+      attr_accessor :inhibit_cleanup
 
-    # The checksum of the RuboCop program running the inspection.
-    def rubocop_checksum
-      ResultCache.source_checksum ||=
-        begin
+      # The checksum of the RuboCop program running the inspection.
+      def source_checksum
+        @source_checksum ||= begin
           digest = Digest::SHA1.new
           rubocop_extra_features
             .select { |path| File.file?(path) }
@@ -184,21 +203,33 @@ module RuboCop
           digest << RuboCop::Version::STRING << RuboCop::AST::Version::STRING
           digest.hexdigest
         end
-    end
+      end
 
-    def digest(path)
-      content = if path.end_with?(*DL_EXTENSIONS)
-                  # Shared libraries often contain timestamps of when
-                  # they were compiled and other non-stable data.
-                  File.basename(path)
-                else
-                  File.binread(path) # mtime not reliable
-                end
-      Zlib.crc32(content).to_s
-    end
+      # Return a hash of the options given at invocation, minus the ones that have
+      # no effect on which offenses and disabled line ranges are found, and thus
+      # don't affect caching.
+      def relevant_options_digest(options)
+        @relevant_options_digest ||= {}
+        @relevant_options_digest[options] ||= begin
+          options = options.reject { |key, _| NON_CHANGING.include?(key) }
+          options.to_s.gsub(/[^a-z]+/i, '_')
+        end
+      end
+
+      private
 
-    def rubocop_extra_features
-      @rubocop_extra_features ||= begin
+      def digest(path)
+        content = if path.end_with?(*DL_EXTENSIONS)
+                    # Shared libraries often contain timestamps of when
+                    # they were compiled and other non-stable data.
+                    File.basename(path)
+                  else
+                    File.binread(path) # mtime not reliable
+                  end
+        Zlib.crc32(content).to_s
+      end
+
+      def rubocop_extra_features
         lib_root = File.join(File.dirname(__FILE__), '..')
         exe_root = File.join(lib_root, '..', 'exe')
 
@@ -216,20 +247,12 @@ module RuboCop
       end
     end
 
-    # Return a hash of the options given at invocation, minus the ones that have
-    # no effect on which offenses and disabled line ranges are found, and thus
-    # don't affect caching.
-    def relevant_options_digest(options)
-      options = options.reject { |key, _| NON_CHANGING.include?(key) }
-      options.to_s.gsub(/[^a-z]+/i, '_')
-    end
-
     # We combine team and options into a single "context" checksum to avoid
     # making file names that are too long for some filesystems to handle.
     # This context is for anything that's not (1) the RuboCop executable
     # checksum or (2) the inspected file checksum.
     def context_checksum(team, options)
-      keys = [team.external_dependency_checksum, relevant_options_digest(options)]
+      keys = [team.external_dependency_checksum, self.class.relevant_options_digest(options)]
       Digest::SHA1.hexdigest(keys.join)
     end
   end

data/lib/rubocop/rspec/cop_helper.rb

@@ -6,6 +6,12 @@ require 'tempfile'
 module CopHelper
   extend RSpec::SharedContext
 
+  @integrated_plugins = false
+
+  class << self
+    attr_accessor :integrated_plugins
+  end
+
   let(:ruby_version) do
     # The minimum version Prism can parse is 3.3.
     ENV['PARSER_ENGINE'] == 'parser_prism' ? 3.3 : RuboCop::TargetRuby::DEFAULT_VERSION
@@ -18,11 +24,13 @@ module CopHelper
 
   before(:all) do
     next if ENV['RUBOCOP_CORE_DEVELOPMENT']
+    next if CopHelper.integrated_plugins
 
     plugins = Gem.loaded_specs.filter_map do |feature_name, feature_specification|
       feature_name if feature_specification.metadata['default_lint_roller_plugin']
     end
     RuboCop::Plugin.integrate_plugins(RuboCop::Config.new, plugins)
+    CopHelper.integrated_plugins = true
   end
 
   def inspect_source(source, file = nil)

data/lib/rubocop/rspec/shared_contexts.rb

@@ -2,8 +2,12 @@
 
 require 'tmpdir'
 
+# Reset cached PathUtil.pwd before each example so that tests using Dir.chdir
+# or stubbing Dir.pwd get a fresh value.
+RSpec.configure { |c| c.before { RuboCop::PathUtil.reset_pwd } }
+
 RSpec.shared_context 'isolated environment' do # rubocop:disable Metrics/BlockLength
-  around do |example|
+  around do |example| # rubocop:disable Metrics/BlockLength
     Dir.mktmpdir do |tmpdir|
       original_home = Dir.home
       original_xdg_config_home = ENV.fetch('XDG_CONFIG_HOME', nil)
@@ -26,12 +30,17 @@ RSpec.shared_context 'isolated environment' do # rubocop:disable Metrics/BlockLe
       begin
         FileUtils.mkdir_p(working_dir)
 
-        Dir.chdir(working_dir) { example.run }
+        Dir.chdir(working_dir) do
+          RuboCop::PathUtil.reset_pwd
+          RuboCop::ResultCache.reset_config_cache
+          example.run
+        end
       ensure
         ENV['HOME'] = original_home
         ENV['XDG_CONFIG_HOME'] = original_xdg_config_home
 
-        RuboCop::FileFinder.root_level = nil
+        RuboCop::ResultCache.reset_config_cache
+        RuboCop::ConfigLoader.clear_options # This also resets RuboCop::FileFinder.root_level
       end
     end
   end
@@ -203,6 +212,27 @@ RSpec.shared_context 'lsp' do
   end
 end
 
+RSpec.shared_context 'with exclude limit tracking' do
+  around do |example|
+    Dir.mktmpdir('rubocop-exclude-limit') do |dir|
+      RuboCop::ExcludeLimit.tmp_dir = Pathname.new(dir)
+      example.run
+    ensure
+      RuboCop::ExcludeLimit.tmp_dir = nil
+    end
+  end
+
+  # Reads exclude_limit values from the tmp files written by ExcludeLimit.
+  # Returns a hash like { 'Max' => 81 } or nil if no values were written.
+  def read_exclude_limit(cop, parameter_name = nil)
+    if parameter_name
+      read_exclude_limit(cop)[parameter_name]
+    else
+      RuboCop::ExcludeLimit.read_limits(cop.class.badge.to_s)
+    end
+  end
+end
+
 RSpec.shared_context 'ruby 2.0' do
   # Prism supports parsing Ruby 3.3+.
   let(:ruby_version) { ENV['PARSER_ENGINE'] == 'parser_prism' ? 3.3 : 2.0 }
@@ -266,6 +296,10 @@ RSpec.shared_context 'ruby 3.4' do
   let(:ruby_version) { 3.4 }
 end
 
-RSpec.shared_context 'ruby 3.5' do
-  let(:ruby_version) { 3.5 }
+RSpec.shared_context 'ruby 4.0' do
+  let(:ruby_version) { 4.0 }
+end
+
+RSpec.shared_context 'ruby 4.1' do
+  let(:ruby_version) { 4.1 }
 end

data/lib/rubocop/rspec/support.rb

@@ -30,5 +30,6 @@ RSpec.configure do |config|
   config.include_context 'ruby 3.2', :ruby32
   config.include_context 'ruby 3.3', :ruby33
   config.include_context 'ruby 3.4', :ruby34
-  config.include_context 'ruby 3.5', :ruby35
+  config.include_context 'ruby 4.0', :ruby40
+  config.include_context 'ruby 4.1', :ruby41
 end

data/lib/rubocop/runner.rb

@@ -62,14 +62,21 @@ module RuboCop
       @errors = []
       @warnings = []
       @aborting = false
+      @inspected_files = []
+      @report_queue = {}
     end
 
     def run(paths)
+      # Compute the cache source checksum once to avoid potential
+      # inconsistencies between workers.
+      ResultCache.source_checksum
+
       target_files = find_target_files(paths)
+      @project_index = ProjectIndexLoader.build_index(target_files) if project_index_enabled?
+
       if @options[:list_target_files]
         list_files(target_files)
       else
-        warm_cache(target_files) if @options[:parallel]
         inspect_files(target_files)
       end
     rescue Interrupt
@@ -86,21 +93,6 @@ module RuboCop
 
     private
 
-    # Warms up the RuboCop cache by forking a suitable number of RuboCop
-    # instances that each inspects its allotted group of files.
-    def warm_cache(target_files)
-      saved_options = @options.dup
-      if target_files.length <= 1
-        puts 'Skipping parallel inspection: only a single file needs inspection' if @options[:debug]
-        return
-      end
-      puts 'Running parallel inspection' if @options[:debug]
-      %i[autocorrect safe_autocorrect].each { |opt| @options[opt] = false }
-      Parallel.each(target_files) { |target_file| file_offenses(target_file) }
-    ensure
-      @options = saved_options
-    end
-
     def find_target_files(paths)
       target_finder = TargetFinder.new(@config_store, @options)
       mode = if @options[:only_recognized_file_types]
@@ -112,12 +104,26 @@ module RuboCop
       target_files.each(&:freeze).freeze
     end
 
-    def inspect_files(files)
-      inspected_files = []
+    def project_index_enabled?
+      return false unless @config_store.for_pwd.for_all_cops['UseProjectIndex']
 
+      unless ProjectIndexLoader.available?
+        ProjectIndexLoader.warn_unavailable
+        return false
+      end
+
+      true
+    end
+
+    def inspect_files(files) # rubocop:disable Metrics/AbcSize
       formatter_set.started(files)
+      file_iterator(files) do |file|
+        offenses = process_file(file)
+        succeeded = offenses.none? { |o| considered_failure?(o) && offense_displayed?(o) }
+        raise Parallel::Break if @options[:fail_fast] && !succeeded
 
-      each_inspected_file(files) { |file| inspected_files << file }
+        [offenses, succeeded]
+      end
     ensure
       # OPTIMIZE: Calling `ResultCache.cleanup` takes time. This optimization
       # mainly targets editors that integrates RuboCop. When RuboCop is run
@@ -125,23 +131,88 @@ module RuboCop
       if files.size > 1 && cached_run?
         ResultCache.cleanup(@config_store, @options[:debug], @options[:cache_root])
       end
-
-      formatter_set.finished(inspected_files.freeze)
+      formatter_set.finished(@inspected_files.freeze)
       formatter_set.close_output_files
     end
 
-    def each_inspected_file(files)
-      files.reduce(true) do |all_passed, file|
-        offenses = process_file(file)
-        yield file
+    def file_iterator(files, &block)
+      all_passed = true
 
-        if offenses.any? { |o| considered_failure?(o) && offense_displayed?(o) }
-          break false if @options[:fail_fast]
+      on_start = ->(file, _index) { file_started(file) }
+      on_finish = lambda do |file, index, (offenses, passed)|
+        all_passed &&= passed
+        finished_report(file, index, offenses)
+      end
 
-          next false
-        end
+      if run_in_parallel?(files)
+        parallel_file_iterator(files, on_start, on_finish, &block)
+      else
+        serial_file_iterator(files, on_start, on_finish, &block)
+      end
+
+      process_remaining_report_queue
+
+      all_passed
+    end
+
+    def finished_report(file, index, offenses)
+      @report_queue[index] = [file, offenses]
+      @next_index_to_report ||= 0
+      while @report_queue.key?(@next_index_to_report)
+        process_report_queue_entry(@next_index_to_report)
+        @next_index_to_report += 1
+      end
+    end
+
+    def process_report_queue_entry(index)
+      file, offenses = @report_queue.delete(index)
+      file_finished(file, offenses)
+    end
+
+    def process_remaining_report_queue
+      @report_queue.keys.sort.each do |index|
+        process_report_queue_entry(index)
+      end
+    end
 
-        all_passed
+    def run_in_parallel?(files)
+      return false if @options[:auto_gen_config]
+      return false unless @options[:parallel]
+
+      if files.size <= 1
+        puts 'Skipping parallel inspection: only a single file needs inspection' if @options[:debug]
+        return false
+      end
+
+      return false if project_index_disables_parallel?
+
+      puts 'Running parallel inspection' if @options[:debug]
+
+      true
+    end
+
+    def project_index_disables_parallel?
+      return false if @project_index.nil? || !Gem.win_platform?
+
+      if @options[:debug]
+        puts 'Skipping parallel inspection: the project index is enabled and parallel ' \
+             'inspection is not yet supported on Windows.'
+      end
+
+      true
+    end
+
+    def parallel_file_iterator(files, on_start, on_finish, &block)
+      Parallel.each(files, start: on_start, finish: on_finish, &block)
+    end
+
+    def serial_file_iterator(files, on_start, on_finish, &block)
+      files.each_with_index do |file, index|
+        on_start.call(file, index)
+        result = yield file
+        on_finish.call(file, index, result)
+      rescue Parallel::Break
+        break
       end
     end
 
@@ -150,16 +221,13 @@ module RuboCop
     end
 
     def process_file(file)
-      file_started(file)
-      offenses = file_offenses(file)
+      file_offenses(file)
     rescue InfiniteCorrectionLoop => e
       raise e if @options[:raise_cop_error]
 
       errors << e
       warn Rainbow(e.message).red
-      offenses = e.offenses.compact.sort.freeze
-    ensure
-      file_finished(file, offenses || [])
+      e.offenses.compact.sort.freeze
     end
 
     def file_offenses(file)
@@ -190,7 +258,7 @@ module RuboCop
 
       if real_run_needed
         offenses = yield
-        save_in_cache(cache, offenses)
+        save_in_cache(cache, offenses) unless Cop::Registry.global.warnings?(file)
       end
 
       offenses
@@ -246,6 +314,7 @@ module RuboCop
     end
 
     def file_finished(file, offenses)
+      @inspected_files << file
       offenses = offenses_to_report(offenses)
       formatter_set.file_finished(file, offenses)
     end
@@ -254,10 +323,6 @@ module RuboCop
       @cached_run ||=
         (@options[:cache] == 'true' ||
          (@options[:cache] != 'false' && @config_store.for_pwd.for_all_cops['UseCache'])) &&
-        # When running --auto-gen-config, there's some processing done in the
-        # cops related to calculating the Max parameters for Metrics cops. We
-        # need to do that processing and cannot use caching.
-        !@options[:auto_gen_config] &&
         # We can't cache results from code which is piped in to stdin
         !@options[:stdin]
     end
@@ -273,7 +338,8 @@ module RuboCop
     end
 
     def do_inspection_loop(file)
-      processed_source = get_processed_source(file)
+      # We can reuse the prism result since the source did not change yet.
+      processed_source = get_processed_source(file, @prism_result)
       # This variable is 2d array used to track corrected offenses after each
       # inspection iteration. This is used to output meaningful infinite loop
       # error message.
@@ -295,7 +361,8 @@ module RuboCop
         # loop if we find any.
         break unless updated_source_file
 
-        processed_source = get_processed_source(file)
+        # Autocorrect has happened, don't use the prism result since it is stale.
+        processed_source = get_processed_source(file, nil)
       end
 
       # Return summary of corrected offenses after all iterations
@@ -344,16 +411,9 @@ module RuboCop
 
     def inspect_file(processed_source, team = mobilize_team(processed_source))
       extracted_ruby_sources = extract_ruby_sources(processed_source)
-      offenses = extracted_ruby_sources.flat_map do |extracted_ruby_source|
-        report = team.investigate(
-          extracted_ruby_source[:processed_source],
-          offset: extracted_ruby_source[:offset],
-          original: processed_source
-        )
-        @errors.concat(team.errors)
-        @warnings.concat(team.warnings)
-        report.offenses
-      end
+      offenses = team.investigate_fragments(extracted_ruby_sources, original: processed_source)
+      @errors.concat(team.errors)
+      @warnings.concat(team.warnings)
       [offenses, team.updated_source_file?]
     end
 
@@ -373,7 +433,15 @@ module RuboCop
 
     def mobilize_team(processed_source)
       config = @config_store.for_file(processed_source.path)
-      Cop::Team.mobilize(mobilized_cop_classes(config), config, @options)
+      team = Cop::Team.mobilize(mobilized_cop_classes(config), config, @options)
+
+      if @project_index
+        team.cops.each do |cop|
+          cop.project_index = @project_index
+        end
+      end
+
+      team
     end
 
     def mobilized_cop_classes(config) # rubocop:disable Metrics/AbcSize
@@ -482,7 +550,7 @@ module RuboCop
     end
 
     # rubocop:disable Metrics/MethodLength
-    def get_processed_source(file)
+    def get_processed_source(file, prism_result)
       config = @config_store.for_file(file)
       ruby_version = config.target_ruby_version
       parser_engine = config.parser_engine
@@ -493,7 +561,7 @@ module RuboCop
                              ruby_version,
                              file,
                              parser_engine: parser_engine,
-                             prism_result: @prism_result
+                             prism_result: prism_result
                            )
                          else
                            begin
@@ -516,8 +584,17 @@ module RuboCop
     # level caching in ResultCache.
     def standby_team(config)
       @team_by_config ||= {}.compare_by_identity
-      @team_by_config[config] ||=
-        Cop::Team.mobilize(mobilized_cop_classes(config), config, @options)
+      @team_by_config[config] ||= begin
+        team = Cop::Team.mobilize(mobilized_cop_classes(config), config, @options)
+
+        if @project_index
+          team.cops.each do |cop|
+            cop.project_index = @project_index
+          end
+        end
+
+        team
+      end
     end
   end
 end

data/lib/rubocop/server/cache.rb

@@ -73,33 +73,12 @@ module RuboCop
           cache_root_dir = if cache_root_path
                              File.join(cache_root_path, 'rubocop_cache')
                            else
-                             cache_root_dir_from_config
+                             CacheConfig.root_dir_from_toplevel_config
                            end
 
           File.expand_path(File.join(cache_root_dir, 'server'))
         end
 
-        def cache_root_dir_from_config
-          CacheConfig.root_dir do
-            # `RuboCop::ConfigStore` has heavy dependencies, this is a lightweight implementation
-            # so that only the necessary `CacheRootDirectory` can be obtained.
-            config_path = ConfigFinder.find_config_path(Dir.pwd)
-            file_contents = File.read(config_path)
-
-            # Returns early if `CacheRootDirectory` is not used before requiring `erb` or `yaml`.
-            next unless file_contents.include?('CacheRootDirectory')
-
-            config_yaml = load_erb_templated_yaml(file_contents)
-
-            # For compatibility with Ruby 3.0 or lower.
-            if Gem::Version.new(Psych::VERSION) < Gem::Version.new('4.0.0')
-              config_yaml == false ? nil : config_yaml
-            end
-
-            config_yaml&.dig('AllCops', 'CacheRootDirectory')
-          end
-        end
-
         def port_path
           dir.join('port')
         end
@@ -135,13 +114,11 @@ module RuboCop
         end
 
         def acquire_lock
-          lock_file = File.open(lock_path, File::CREAT)
-          # flock returns 0 if successful, and false if not.
-          flock_result = lock_file.flock(File::LOCK_EX | File::LOCK_NB)
-          yield flock_result != false
-        ensure
-          lock_file.flock(File::LOCK_UN)
-          lock_file.close
+          File.open(lock_path, File::CREAT) do |lock_file|
+            # flock returns 0 if successful, and false if not.
+            flock_result = lock_file.flock(File::LOCK_EX | File::LOCK_NB)
+            yield flock_result != false
+          end
         end
 
         def write_port_and_token_files(port:, token:)

data/lib/rubocop/server/core.rb

@@ -45,6 +45,8 @@ module RuboCop
         write_port_and_token_files
 
         pid = fork do
+          # NOTE: As of Ruby 4.0.0, ZJIT is still under development, while YJIT is production-ready,
+          # so support for ZJIT is deferred.
           if defined?(RubyVM::YJIT.enable)
             RubyVM::YJIT.enable
           end
@@ -55,6 +57,12 @@ module RuboCop
         end
 
         Process.waitpid(pid)
+
+        # The daemon writes its pid file asynchronously after forking, so wait until
+        # the server is actually running before returning. This prevents a race where
+        # a subsequent command (e.g. `--restart-server`) observes an inconsistent
+        # state right after `--start-server` returns.
+        Server.wait_for_running_status!(true)
       end
 
       def write_port_and_token_files