knapsack_pro 9.1.0 → 9.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3968c8fcf00e58b3ba539fb37bd8458455c9779b76e2cb118dd09be8caa3cc5
-  data.tar.gz: fea6294e3ec2bd6edb7c18dae2a6c42c06cd14a3c28e4cb5a543862f4bb6326b
+  metadata.gz: 69f4cd183de10604ce425379105354fd37da56929568c6bac466d682dc893088
+  data.tar.gz: 3c9c33a0aed5050e6d6582ff66550d8497b833383b1b34c09e622dd8f57bc6b8
 SHA512:
-  metadata.gz: eb649c4dd8c047d5455a487767c7ce30d6424e340673620c4b6365f0a2cb8aedc870a83d304f1ca024fb3d3e39c2ccf003536262f986324326814915b8bdfccf
-  data.tar.gz: 5aee2305c85f6cd6760d75ae456a12101305dc5725cdc2b2641c13d17cf1004ad5e90168256352665c884c1bc58f702289f65a808cc3c7383667b5a88a0f52c1
+  metadata.gz: bac25fed3f1e309e57f21bbce67391ed2d638d8484c91ce99900aa9ab22bbebde61bf0a968e9bf208f45d9d684574cb8a0b9fd549a0f5875c036b4c0e6aeec30
+  data.tar.gz: 35f0c05d901993faa07ceeffc211e596b693aa852671e9a04aa12ca845ef78bb39e332b8011b846a93be6cab17f803197f2e7a008fe9f7d9140ba9df75240561

data/lib/knapsack_pro/adapters/base_adapter.rb

@@ -15,26 +15,10 @@ module KnapsackPro
         false
       end
 
-      def self.test_file_cases_for(slow_test_files)
+      def self.calculate_slow_id_paths
         raise NotImplementedError
       end
 
-      def self.slow_test_file?(adapter_class, test_file_path)
-        @slow_test_file_paths ||=
-          begin
-            slow_test_files =
-              if KnapsackPro::Config::Env.slow_test_file_pattern
-                KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
-              else
-                # get slow test files from JSON file based on data from API
-                KnapsackPro::SlowTestFileDeterminer.read_from_json_report
-              end
-            KnapsackPro::TestFilePresenter.paths(slow_test_files)
-          end
-        clean_path = KnapsackPro::TestFileCleaner.clean(test_file_path)
-        @slow_test_file_paths.include?(clean_path)
-      end
-
       def self.bind
         adapter = new
         adapter.bind

data/lib/knapsack_pro/adapters/rspec_adapter.rb

@@ -20,22 +20,17 @@ module KnapsackPro
         true
       end
 
-      def self.test_file_cases_for(slow_test_files)
-        KnapsackPro.logger.info("Generating RSpec test examples JSON report for slow test files to prepare it to be split by test examples (by individual test cases). Thanks to that, a single slow test file can be split across parallel CI nodes. Analyzing #{slow_test_files.size} slow test files.")
-
-        # generate the RSpec JSON report in a separate process to not pollute the RSpec state
+      def self.calculate_slow_id_paths
+        # Shell out not to pollute the RSpec state
         cmd = [
           'RACK_ENV=test',
           'RAILS_ENV=test',
           KnapsackPro::Config::Env.rspec_test_example_detector_prefix,
           'rake knapsack_pro:rspec_test_example_detector',
         ].join(' ')
-        unless Kernel.system(cmd)
-          raise "Could not generate JSON report for RSpec. Rake task failed when running #{cmd}"
-        end
+        raise "Failed to calculate Split by Test Examples: #{cmd}" unless Kernel.system(cmd)
 
-        # read the JSON report
-        KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new.test_file_example_paths
+        KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new.slow_id_paths!
       end
 
       def self.has_format_option?(cli_args)
@@ -86,6 +81,13 @@ module KnapsackPro
         !id.nil?
       end
 
+      def self.concat_paths(test_files, id_paths)
+        paths = KnapsackPro::TestFilePresenter.paths(test_files)
+        file_paths = id_paths.map { |id_path| parse_file_path(id_path) }
+        acc = paths + id_paths - file_paths
+        KnapsackPro::TestFilePresenter.test_files(acc)
+      end
+
       def self.rails_helper_exists?(test_dir)
         File.exist?("#{test_dir}/rails_helper.rb")
       end

data/lib/knapsack_pro/build_distribution_fetcher.rb

@@ -7,10 +7,6 @@ module KnapsackPro
         @response = response
       end
 
-      def time_execution
-        response.fetch('time_execution')
-      end
-
       def test_files
         response.fetch('test_files')
       end
@@ -20,12 +16,6 @@ module KnapsackPro
       attr_reader :response
     end
 
-    def self.call
-      new.call
-    end
-
-    # get test files and time execution for last build distribution matching:
-    # branch, node_total, node_index
     def call
       connection = KnapsackPro::Client::Connection.new(build_action)
       response = connection.call
@@ -33,11 +23,8 @@ module KnapsackPro
         raise ArgumentError.new(response) if connection.errors?
         BuildDistributionEntity.new(response)
       else
-        KnapsackPro.logger.warn("Slow test files fallback behaviour started. We could not connect with Knapsack Pro API to fetch last CI build test files that are needed to determine slow test files. No test files will be split by test cases. It means all test files will be split by the whole test files as if split by test cases would be disabled #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}")
-        BuildDistributionEntity.new({
-          'time_execution' => 0.0,
-          'test_files' => [],
-        })
+        KnapsackPro.logger.warn("Failed to fetch slow test files. Split by Test Examples disabled. See: #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}")
+        BuildDistributionEntity.new({ 'time_execution' => 0.0, 'test_files' => [] })
       end
     end
 
@@ -48,12 +35,29 @@ module KnapsackPro
     end
 
     def build_action
-      KnapsackPro::Client::API::V1::BuildDistributions.last(
+      request_hash = {
         commit_hash: repository_adapter.commit_hash,
         branch: repository_adapter.branch,
         node_total: KnapsackPro::Config::Env.ci_node_total,
-        node_index: KnapsackPro::Config::Env.ci_node_index,
-      )
+        node_index: KnapsackPro::Config::Env.ci_node_index
+      }.merge(additional_params)
+
+      KnapsackPro::Client::API::V1::BuildDistributions.last(request_hash)
+    end
+
+    def additional_params
+      {}
+    end
+  end
+
+  class OptimizedBuildDistributionFetcher < BuildDistributionFetcher
+    private
+
+    def additional_params
+      {
+        node_build_id: KnapsackPro::Config::Env.ci_node_build_id,
+        none_if_queue_initialized: true
+      }
     end
   end
 end

data/lib/knapsack_pro/client/api/v1/build_distributions.rb

@@ -35,16 +35,11 @@ module KnapsackPro
               )
             end
 
-            def last(args)
+            def last(request_hash)
               action_class.new(
                 endpoint_path: '/v1/build_distributions/last',
                 http_method: :get,
-                request_hash: {
-                  :commit_hash => args.fetch(:commit_hash),
-                  :branch => args.fetch(:branch),
-                  :node_total => args.fetch(:node_total),
-                  :node_index => args.fetch(:node_index),
-                }
+                request_hash: request_hash
               )
             end
           end

data/lib/knapsack_pro/client/api/v1/queues.rb

@@ -36,6 +36,33 @@ module KnapsackPro
                 request_hash: request_hash
               )
             end
+
+            def initialize(paths)
+              git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
+              repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
+
+              request_hash = {
+                attempt_connect_to_queue: false,
+                branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
+                build_author: git_adapter.build_author,
+                can_initialize_queue: true,
+                commit_authors: git_adapter.commit_authors,
+                commit_hash: repository_adapter.commit_hash,
+                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split,
+                node_build_id: KnapsackPro::Config::Env.ci_node_build_id,
+                node_index: KnapsackPro::Config::Env.ci_node_index,
+                node_total: KnapsackPro::Config::Env.ci_node_total,
+                skip_pull: true,
+                test_files: KnapsackPro::Crypto::Encryptor.call(paths),
+                user_seat: KnapsackPro::Config::Env.masked_user_seat,
+              }
+
+              action_class.new(
+                endpoint_path: '/v1/queues/queue',
+                http_method: :post,
+                request_hash: request_hash
+              )
+            end
           end
         end
       end

data/lib/knapsack_pro/config/env.rb

@@ -67,6 +67,14 @@ module KnapsackPro
           ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN']
         end
 
+        def slow_test_file_threshold
+          ENV.fetch('KNAPSACK_PRO_SLOW_TEST_FILE_THRESHOLD', nil)&.to_f
+        end
+
+        def slow_test_file_threshold?
+          !!slow_test_file_threshold
+        end
+
         def test_file_exclude_pattern
           ENV['KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN']
         end
@@ -188,14 +196,6 @@ module KnapsackPro
           ENV.fetch('KNAPSACK_PRO_RSPEC_TEST_EXAMPLE_DETECTOR_PREFIX', 'bundle exec')
         end
 
-        def slow_test_file_threshold
-          ENV.fetch('KNAPSACK_PRO_SLOW_TEST_FILE_THRESHOLD', nil)&.to_f
-        end
-
-        def slow_test_file_threshold?
-          !!slow_test_file_threshold
-        end
-
         def test_suite_token
           env_name = 'KNAPSACK_PRO_TEST_SUITE_TOKEN'
           ENV[env_name] || raise("Missing environment variable #{env_name}. You should set environment variable like #{env_name}_RSPEC (note there is suffix _RSPEC at the end). knapsack_pro gem will set #{env_name} based on #{env_name}_RSPEC value. If you use other test runner than RSpec then use proper suffix.")

data/lib/knapsack_pro/queue_allocator.rb

@@ -141,8 +141,9 @@ module KnapsackPro
       end
     end
 
+    # Run file paths to guarantee at-least-once execution across CI nodes.
     def fallback_test_files(executed_test_files)
-      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_suite.fallback_test_files, ci_node_total)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_suite.all_test_files_to_run, ci_node_total)
       test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
       KnapsackPro::TestFilePresenter.paths(test_files_for_node_index) - executed_test_files
     end

data/lib/knapsack_pro/queue_initializer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  module RSpec
+    class QueueInitializer
+      def call(args)
+        slow_id_paths = KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new.calculate_slow_id_paths(args.to_s)
+        all_test_files_to_run = KnapsackPro::TestSuite.new(KnapsackPro::Adapters::RSpecAdapter).all_test_files_to_run
+        paths = KnapsackPro::Adapters::RSpecAdapter.concat_paths(all_test_files_to_run, slow_id_paths)
+
+        raise 'No paths to run' if paths.empty?
+        action = KnapsackPro::Client::API::V1::Queues.initialize(paths)
+        connection = KnapsackPro::Client::Connection.new(action)
+        response = connection.call
+        return unless response.key?('url') # Race to initialize lost to another parallel node
+
+        KnapsackPro.logger.info "Build URL: #{response.fetch('url')}"
+      end
+    end
+  end
+end

data/lib/knapsack_pro/regular_allocator.rb

@@ -131,8 +131,9 @@ module KnapsackPro
       end
     end
 
+    # Run file paths to guarantee at-least-once execution across CI nodes.
     def fallback_test_files
-      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_suite.fallback_test_files, ci_node_total)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_suite.all_test_files_to_run, ci_node_total)
       test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
       KnapsackPro::TestFilePresenter.paths(test_files_for_node_index)
     end

data/lib/knapsack_pro/rspec_slow_test_file_finder.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  class RSpecSlowTestFileFinder
+    def initialize(build_distribution_fetcher)
+      @build_distribution_fetcher = build_distribution_fetcher
+    end
+
+    def call
+      if KnapsackPro::Config::Env.test_files_encrypted?
+        raise "Split by test cases is not possible when you have enabled test file names encryption ( #{KnapsackPro::Urls::ENCRYPTION} ). You need to disable encryption with KNAPSACK_PRO_TEST_FILES_ENCRYPTED=false in order to use split by test cases #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}"
+      end
+
+      test_files_from_api = @build_distribution_fetcher.call.test_files
+      merged_test_files_from_api = KnapsackPro::TestCaseMergers::RSpecMerger.new(test_files_from_api).call
+      test_files_existing_on_disk = KnapsackPro::TestFileFinder.select_test_files_that_can_be_run(KnapsackPro::Adapters::RSpecAdapter, merged_test_files_from_api)
+      KnapsackPro::SlowTestFileDeterminer.call(test_files_existing_on_disk)
+    end
+  end
+end

data/lib/knapsack_pro/slow_test_file_determiner.rb

@@ -18,27 +18,5 @@ module KnapsackPro
         execution_time >= KnapsackPro::Config::Env.slow_test_file_threshold
       end
     end
-
-    def self.save_to_json_report(test_files)
-      KnapsackPro::Config::TempFiles.ensure_temp_directory_exists!
-      FileUtils.mkdir_p(report_dir)
-      File.write(report_path, test_files.to_json)
-    end
-
-    def self.read_from_json_report
-      raise "The report with slow test files has not been generated yet. If you have enabled split by test cases #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES} and you see this error it means that your tests accidentally cleaned up the .knapsack_pro directory. Please do not remove this directory during tests runtime!" unless File.exist?(report_path)
-      slow_test_files_json_report = File.read(report_path)
-      JSON.parse(slow_test_files_json_report)
-    end
-
-    private
-
-    def self.report_path
-      "#{report_dir}/slow_test_files_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
-    end
-
-    def self.report_dir
-      "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/slow_test_file_determiner"
-    end
   end
 end

data/lib/knapsack_pro/test_case_detectors/rspec_test_example_detector.rb

@@ -3,87 +3,80 @@
 module KnapsackPro
   module TestCaseDetectors
     class RSpecTestExampleDetector
-      def generate_json_report(rspec_args)
-        raise "The internal KNAPSACK_PRO_RSPEC_OPTIONS environment variable is unset. Ensure it is not overridden accidentally. Otherwise, please report this as a bug: #{KnapsackPro::Urls::SUPPORT}" if rspec_args.nil?
-
-        require 'rspec/core'
-
-        cli_format =
-          if Gem::Version.new(::RSpec::Core::Version::STRING) < Gem::Version.new('3.6.0')
-            require_relative '../formatters/rspec_json_formatter'
-            ['--format', KnapsackPro::Formatters::RSpecJsonFormatter.to_s]
-          else
-            ['--format', 'json']
-          end
-
-        ensure_report_dir_exists
-        remove_old_json_report
-
-        test_file_entities = slow_test_files
-
-        if test_file_entities.empty?
-          no_examples_json = { examples: [] }.to_json
-          File.write(report_path, no_examples_json)
-          return
-        end
+      def dry_run_to_file(rspec_args, slow_test_files = slow_test_files(KnapsackPro::BuildDistributionFetcher.new))
+        KnapsackPro::Config::TempFiles.ensure_temp_directory_exists!
+        FileUtils.mkdir_p(File.dirname(report_path))
+        File.delete(report_path) if File.exist?(report_path)
+        return File.write(report_path, { examples: [] }.to_json) if slow_test_files.empty?
 
+        KnapsackPro.logger.info("Calculating Split by Test Examples. Analyzing #{slow_test_files.size} slow test files.")
         args = (rspec_args || '').split
         cli_args_without_formatters = KnapsackPro::Adapters::RSpecAdapter.remove_formatters(args)
-
-        # Apply a --format option which overrides formatters from the RSpec custom option files like `.rspec`.
         cli_args = cli_args_without_formatters + cli_format + [
           '--dry-run',
           '--out', report_path,
           '--default-path', test_dir
-        ] + KnapsackPro::TestFilePresenter.paths(test_file_entities)
-        exit_code = begin
-          options = ::RSpec::Core::ConfigurationOptions.new(cli_args)
-          ::RSpec::Core::Runner.new(options).run($stderr, $stdout)
-        rescue SystemExit => e
-          e.status
-        end
-
+        ] + KnapsackPro::TestFilePresenter.paths(slow_test_files)
+        exit_code = dry_run(cli_args)
         return if exit_code.zero?
 
         report.fetch('messages', []).each { |message| puts message }
         command = (['bundle exec rspec'] + cli_args).join(' ')
-        KnapsackPro.logger.error("Failed to generate the slow test files report: #{command}")
+        KnapsackPro.logger.error("Failed to calculate Split by Test Examples: #{command}")
         exit exit_code
       end
 
-      def test_file_example_paths
+      def calculate_slow_id_paths(rspec_args)
+        dry_run_to_file(rspec_args, slow_test_files(KnapsackPro::OptimizedBuildDistributionFetcher.new))
+        slow_id_paths!
+      end
+
+      def slow_id_paths!
         raise "No report found at #{report_path}" unless File.exist?(report_path)
 
-        json_report = File.read(report_path)
-        hash_report = JSON.parse(json_report)
-        hash_report
+        JSON.parse(File.read(report_path))
           .fetch('examples')
-          .map { |e| e.fetch('id') }
-          .map { |path_with_example_id| test_file_hash_for(path_with_example_id) }
+          .map { |example| TestFileCleaner.clean(example.fetch('id')) }
       end
 
-      def slow_test_files
-        if KnapsackPro::Config::Env.slow_test_file_pattern
-          KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+      private
+
+      # Apply a --format option which overrides formatters from the RSpec custom option files like `.rspec`.
+      def cli_format
+        require 'rspec/core'
+
+        if Gem::Version.new(::RSpec::Core::Version::STRING) < Gem::Version.new('3.6.0')
+          require_relative '../formatters/rspec_json_formatter'
+          ['--format', KnapsackPro::Formatters::RSpecJsonFormatter.to_s]
         else
-          # read slow test files from JSON file on disk that was generated
-          # by lib/knapsack_pro/base_allocator_builder.rb
-          KnapsackPro::SlowTestFileDeterminer.read_from_json_report
+          ['--format', 'json']
         end
       end
 
-      private
+      def dry_run(cli_args)
+        require 'rspec/core'
 
-      def report_dir
-        "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/test_case_detectors/rspec"
+        options = ::RSpec::Core::ConfigurationOptions.new(cli_args)
+        ::RSpec::Core::Runner.new(options).run($stderr, $stdout)
+      rescue SystemExit => e
+        e.status
       end
 
       def report_path
-        "#{report_dir}/rspec_dry_run_json_report_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+        "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/test_case_detectors/rspec/rspec_dry_run_json_report_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+      end
+
+      def slow_test_files(build_distribution_fetcher)
+        if KnapsackPro::Config::Env.slow_test_file_pattern
+          KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+        else
+          KnapsackPro::RSpecSlowTestFileFinder.new(build_distribution_fetcher).call
+        end
       end
 
       def report
         return {} unless File.exist?(report_path)
+
         JSON.parse(File.read(report_path))
       end
 
@@ -94,25 +87,6 @@ module KnapsackPro
       def test_dir
         KnapsackPro::Config::Env.test_dir || KnapsackPro::TestFilePattern.test_dir(adapter_class)
       end
-
-      def test_file_pattern
-        KnapsackPro::TestFilePattern.call(adapter_class)
-      end
-
-      def ensure_report_dir_exists
-        KnapsackPro::Config::TempFiles.ensure_temp_directory_exists!
-        FileUtils.mkdir_p(report_dir)
-      end
-
-      def remove_old_json_report
-        File.delete(report_path) if File.exist?(report_path)
-      end
-
-      def test_file_hash_for(test_file_path)
-        {
-          'path' => TestFileCleaner.clean(test_file_path)
-        }
-      end
     end
   end
 end

data/lib/knapsack_pro/test_case_mergers/rspec_merger.rb

@@ -2,44 +2,32 @@
 
 module KnapsackPro
   module TestCaseMergers
-    class RSpecMerger < BaseMerger
+    class RSpecMerger
+      def initialize(test_files)
+        @test_files = test_files
+      end
+
       def call
-        all_test_files_hash = {}
-        merged_test_file_examples_hash = {}
+        file_paths = {}
+        id_paths = {}
 
-        test_files.each do |test_file|
-          path = test_file.fetch('path')
-          test_file_path = extract_test_file_path(path)
+        @test_files.each do |test_file|
+          raw_path = test_file.fetch('path')
+          file_path = KnapsackPro::Adapters::RSpecAdapter.parse_file_path(raw_path)
 
-          if KnapsackPro::Adapters::RSpecAdapter.id_path?(path)
-            merged_test_file_examples_hash[test_file_path] ||= 0.0
-            merged_test_file_examples_hash[test_file_path] += test_file.fetch('time_execution')
+          if KnapsackPro::Adapters::RSpecAdapter.id_path?(raw_path)
+            id_paths[file_path] ||= 0.0
+            id_paths[file_path] += test_file.fetch('time_execution')
           else
-            all_test_files_hash[test_file_path] = test_file.fetch('time_execution')
+            file_paths[file_path] = test_file.fetch('time_execution') # may be nil
           end
         end
 
-        merged_test_file_examples_hash.each do |path, time_execution|
-          all_test_files_hash[path] = [time_execution, all_test_files_hash[path]].compact.max
-        end
-
-        merged_test_files = []
-        all_test_files_hash.each do |path, time_execution|
-          merged_test_files << {
-            'path' => path,
-            'time_execution' => time_execution
-          }
-        end
-        merged_test_files
-      end
-
-      private
-
-      # path - can be:
-      # test file path: spec/a_spec.rb
-      # or test example path: spec/a_spec.rb[1:1]
-      def extract_test_file_path(path)
-        path.gsub(/\.rb\[.+\]$/, '.rb')
+        file_paths
+          .merge(id_paths) { |_, v1, v2| [v1, v2].compact.max }
+          .map do |file_path, time_execution|
+            { 'path' => file_path, 'time_execution' => time_execution }
+          end
       end
     end
   end

data/lib/knapsack_pro/test_file_finder.rb

@@ -6,41 +6,22 @@ module KnapsackPro
       new(test_file_pattern, test_file_list_enabled).call
     end
 
-    # finds slow test files on disk based on ENV patterns
-    # returns example: [{ 'path' => 'a_spec.rb' }]
     def self.slow_test_files_by_pattern(adapter_class)
       raise 'KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN is not defined' unless KnapsackPro::Config::Env.slow_test_file_pattern
 
       test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
-      test_file_entities = call(test_file_pattern)
-
-      slow_test_file_entities = call(KnapsackPro::Config::Env.slow_test_file_pattern, test_file_list_enabled: false)
-
-      # slow test files (KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN)
-      # should be subset of test file pattern (KNAPSACK_PRO_TEST_FILE_PATTERN)
-      slow_test_file_entities & test_file_entities
+      scheduled_test_files = call(test_file_pattern)
+      slow_test_files = call(KnapsackPro::Config::Env.slow_test_file_pattern, test_file_list_enabled: false)
+      scheduled_test_files & slow_test_files
     end
 
-    # Args:
-    #   test_file_entities_to_run - it can be list of slow test files that you want to run
-    # Return:
-    #   subset of test_file_entities_to_run that are present on disk and it is subset of tests matching pattern KNAPSACK_PRO_TEST_FILE_PATTERN
-    #   Thanks to that we can select only slow test files that are within list of test file pattern we want to run tests for
-    def self.select_test_files_that_can_be_run(adapter_class, test_file_entities_to_run)
+    def self.select_test_files_that_can_be_run(adapter_class, candidate_test_files)
       test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
-      test_file_entities = call(test_file_pattern)
-
-      test_file_paths_existing_on_disk = KnapsackPro::TestFilePresenter.paths(test_file_entities)
-
-      selected_test_files = []
-
-      test_file_entities_to_run.each do |test_file_entity|
-        if test_file_paths_existing_on_disk.include?(test_file_entity.fetch('path'))
-          selected_test_files << test_file_entity
-        end
-      end
-
-      selected_test_files
+      scheduled_test_files = call(test_file_pattern)
+      scheduled_paths = KnapsackPro::TestFilePresenter.paths(scheduled_test_files)
+      candidate_paths = KnapsackPro::TestFilePresenter.paths(candidate_test_files)
+      paths = scheduled_paths & candidate_paths
+      candidate_test_files.filter { |test_file| paths.include? test_file.fetch('path') }
     end
 
     def initialize(test_file_pattern, test_file_list_enabled)
@@ -49,18 +30,16 @@ module KnapsackPro
     end
 
     def call
-      test_file_hashes = []
-      test_files.each do |test_file_path|
-        test_file_hashes << test_file_hash_for(test_file_path)
+      file_paths.map do |file_path|
+        { 'path' => TestFileCleaner.clean(file_path) }
       end
-      test_file_hashes
     end
 
     private
 
     attr_reader :test_file_pattern, :test_file_list_enabled
 
-    def test_files
+    def file_paths
       if test_file_list_enabled && KnapsackPro::Config::Env.test_file_list
         return KnapsackPro::Config::Env.test_file_list.split(',').map(&:strip)
       end
@@ -69,22 +48,16 @@ module KnapsackPro
         return File.read(KnapsackPro::Config::Env.test_file_list_source_file).split(/\n/)
       end
 
-      test_file_paths = Dir.glob(test_file_pattern).uniq
+      included_paths = Dir.glob(test_file_pattern).uniq
 
-      excluded_test_file_paths =
+      excluded_paths =
         if KnapsackPro::Config::Env.test_file_exclude_pattern
           Dir.glob(KnapsackPro::Config::Env.test_file_exclude_pattern).uniq
         else
           []
         end
 
-      (test_file_paths - excluded_test_file_paths).sort
-    end
-
-    def test_file_hash_for(test_file_path)
-      {
-        'path' => TestFileCleaner.clean(test_file_path)
-      }
+      (included_paths - excluded_paths).sort
     end
   end
 end

data/lib/knapsack_pro/test_file_presenter.rb

@@ -12,5 +12,9 @@ module KnapsackPro
     def self.paths(test_files)
       test_files.map { |t| t['path'] }
     end
+
+    def self.test_files(paths)
+      paths.map { |path| { 'path' => path } }
+    end
   end
 end

data/lib/knapsack_pro/test_suite.rb

@@ -8,8 +8,6 @@ module KnapsackPro
       @adapter_class = adapter_class
     end
 
-    # Detect test files present on the disk that should be run.
-    # This may include fast test files + slow test files split by test cases.
     def calculate_test_files
       return @result if defined?(@result)
 
@@ -17,47 +15,22 @@ module KnapsackPro
         return @result = Result.new(all_test_files_to_run, true)
       end
 
-      slow_test_files, quick =
-        if slow_test_file_pattern
-          [KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class), true]
-        else
-          [KnapsackPro::SlowTestFileFinder.call(adapter_class), false]
-        end
-
-      KnapsackPro.logger.debug("Detected #{slow_test_files.size} slow test files: #{slow_test_files.inspect}")
-
-      if slow_test_files.empty?
-        return @result = Result.new(all_test_files_to_run, quick)
+      if KnapsackPro::Config::Env.slow_test_file_pattern
+        slow_test_files = KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+        return @result = Result.new(all_test_files_to_run, true) if slow_test_files.empty?
       end
 
-      test_file_cases = adapter_class.test_file_cases_for(slow_test_files)
-
-      fast_files_and_cases_for_slow_tests = KnapsackPro::TestFilesWithTestCasesComposer.call(all_test_files_to_run, slow_test_files, test_file_cases)
-
-      @result = Result.new(fast_files_and_cases_for_slow_tests, false)
+      slow_id_paths = adapter_class.calculate_slow_id_paths
+      test_files = adapter_class.concat_paths(all_test_files_to_run, slow_id_paths)
+      @result = Result.new(test_files, false)
     end
 
-    # In Fallback Mode, we always want to run whole test files (not split by
-    # test cases) to guarantee that each test will be executed at least once
-    # across parallel CI nodes.
-    def fallback_test_files
-      all_test_files_to_run
+    def all_test_files_to_run
+      @all_test_files_to_run ||= KnapsackPro::TestFileFinder.call(TestFilePattern.call(adapter_class))
     end
 
     private
 
     attr_reader :adapter_class
-
-    def all_test_files_to_run
-      @all_test_files_to_run ||= KnapsackPro::TestFileFinder.call(test_file_pattern)
-    end
-
-    def test_file_pattern
-      TestFilePattern.call(adapter_class)
-    end
-
-    def slow_test_file_pattern
-      KnapsackPro::Config::Env.slow_test_file_pattern
-    end
   end
 end

data/lib/knapsack_pro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KnapsackPro
-  VERSION = '9.1.0'
+  VERSION = '9.2.0'
 end

data/lib/knapsack_pro.rb

@@ -58,12 +58,10 @@ require_relative 'knapsack_pro/regular_allocator'
 require_relative 'knapsack_pro/queue_allocator'
 require_relative 'knapsack_pro/mask_string'
 require_relative 'knapsack_pro/test_suite'
-require_relative 'knapsack_pro/test_case_mergers/base_merger'
 require_relative 'knapsack_pro/test_case_mergers/rspec_merger'
 require_relative 'knapsack_pro/build_distribution_fetcher'
 require_relative 'knapsack_pro/slow_test_file_determiner'
-require_relative 'knapsack_pro/slow_test_file_finder'
-require_relative 'knapsack_pro/test_files_with_test_cases_composer'
+require_relative 'knapsack_pro/rspec_slow_test_file_finder'
 require_relative 'knapsack_pro/base_allocator_builder'
 require_relative 'knapsack_pro/regular_allocator_builder'
 require_relative 'knapsack_pro/queue_allocator_builder'

data/lib/tasks/queue/rspec.rake

@@ -12,5 +12,17 @@ namespace :knapsack_pro do
       Rake::Task.clear
       KnapsackPro::Runners::Queue::RSpecRunner.run(args[:rspec_args])
     end
+
+    namespace :rspec do
+      desc 'Initialize the test queue to be consumed later.'
+      task :initialize, [:rspec_args] do |_, args|
+        require_relative '../../knapsack_pro/queue_initializer'
+
+        ENV.delete('SPEC_OPTS') # Ignore `SPEC_OPTS` to not affect the RSpec execution within this rake task
+        ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_rspec
+
+        KnapsackPro::RSpec::QueueInitializer.new.call(args[:rspec_args].to_s)
+      end
+    end
   end
 end

data/lib/tasks/rspec.rake

@@ -7,12 +7,16 @@ namespace :knapsack_pro do
     KnapsackPro::Runners::RSpecRunner.run(args[:rspec_args])
   end
 
-  desc "Generate JSON report for test suite based on default test pattern or based on defined pattern with ENV vars"
+  # private
   task :rspec_test_example_detector do
-    # ignore the `SPEC_OPTS` options to not affect RSpec execution within this rake task
+    key = 'KNAPSACK_PRO_RSPEC_OPTIONS'
+    raise "The internal #{key} environment variable is unset. Ensure it is not overridden accidentally. Otherwise, please report this as a bug: #{KnapsackPro::Urls::SUPPORT}" if ENV[key].nil?
+
+    # Ignore `SPEC_OPTS` to not affect the RSpec execution within this rake task
     ENV.delete('SPEC_OPTS')
 
-    detector = KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new
-    detector.generate_json_report(ENV['KNAPSACK_PRO_RSPEC_OPTIONS'])
+    KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector
+      .new
+      .dry_run_to_file(ENV[key])
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 9.1.0
+  version: 9.2.0
 platform: ruby
 authors:
 - ArturT
@@ -111,16 +111,16 @@ dependencies:
   name: rspec-its
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: spinach
   requirement: !ruby/object:Gem::Requirement
@@ -250,6 +250,7 @@ files:
 - lib/knapsack_pro/queue.rb
 - lib/knapsack_pro/queue_allocator.rb
 - lib/knapsack_pro/queue_allocator_builder.rb
+- lib/knapsack_pro/queue_initializer.rb
 - lib/knapsack_pro/railtie.rb
 - lib/knapsack_pro/regular_allocator.rb
 - lib/knapsack_pro/regular_allocator_builder.rb
@@ -258,6 +259,7 @@ files:
 - lib/knapsack_pro/repository_adapters/base_adapter.rb
 - lib/knapsack_pro/repository_adapters/env_adapter.rb
 - lib/knapsack_pro/repository_adapters/git_adapter.rb
+- lib/knapsack_pro/rspec_slow_test_file_finder.rb
 - lib/knapsack_pro/runners/base_runner.rb
 - lib/knapsack_pro/runners/cucumber_runner.rb
 - lib/knapsack_pro/runners/minitest_runner.rb
@@ -269,16 +271,13 @@ files:
 - lib/knapsack_pro/runners/spinach_runner.rb
 - lib/knapsack_pro/runners/test_unit_runner.rb
 - lib/knapsack_pro/slow_test_file_determiner.rb
-- lib/knapsack_pro/slow_test_file_finder.rb
 - lib/knapsack_pro/task_loader.rb
 - lib/knapsack_pro/test_case_detectors/rspec_test_example_detector.rb
-- lib/knapsack_pro/test_case_mergers/base_merger.rb
 - lib/knapsack_pro/test_case_mergers/rspec_merger.rb
 - lib/knapsack_pro/test_file_cleaner.rb
 - lib/knapsack_pro/test_file_finder.rb
 - lib/knapsack_pro/test_file_pattern.rb
 - lib/knapsack_pro/test_file_presenter.rb
-- lib/knapsack_pro/test_files_with_test_cases_composer.rb
 - lib/knapsack_pro/test_flat_distributor.rb
 - lib/knapsack_pro/test_suite.rb
 - lib/knapsack_pro/tracker.rb
@@ -319,7 +318,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Knapsack Pro splits tests across parallel CI nodes and ensures each parallel
   job finish work at a similar time.

data/lib/knapsack_pro/slow_test_file_finder.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module KnapsackPro
-  class SlowTestFileFinder
-    # Get recorded test files from API.
-    # Find slow tests among them that are still present on the disk.
-    # Save slow test files in json file on the disk.
-    # Returns slow test files.
-    def self.call(adapter_class)
-      if KnapsackPro::Config::Env.test_files_encrypted?
-        raise "Split by test cases is not possible when you have enabled test file names encryption ( #{KnapsackPro::Urls::ENCRYPTION} ). You need to disable encryption with KNAPSACK_PRO_TEST_FILES_ENCRYPTED=false in order to use split by test cases #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}"
-      end
-
-      # get list of recorded test files for last CI Build
-      build_distribution_entity = KnapsackPro::BuildDistributionFetcher.call
-      test_files_from_api = build_distribution_entity.test_files
-
-      merged_test_files_from_api = KnapsackPro::TestCaseMergers::BaseMerger.call(adapter_class, test_files_from_api)
-
-      test_files_existing_on_disk = KnapsackPro::TestFileFinder.select_test_files_that_can_be_run(adapter_class, merged_test_files_from_api)
-
-      slow_test_files = KnapsackPro::SlowTestFileDeterminer.call(test_files_existing_on_disk)
-
-      KnapsackPro::SlowTestFileDeterminer.save_to_json_report(slow_test_files)
-
-      slow_test_files
-    end
-  end
-end

data/lib/knapsack_pro/test_case_mergers/base_merger.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module KnapsackPro
-  module TestCaseMergers
-    class BaseMerger
-      # values must be string to avoid circular dependency problem during loading files
-      ADAPTER_TO_MERGER_MAP = {
-        KnapsackPro::Adapters::RSpecAdapter => 'KnapsackPro::TestCaseMergers::RSpecMerger',
-      }
-
-      def self.call(adapter_class, test_files)
-        merger_class =
-          ADAPTER_TO_MERGER_MAP[adapter_class] ||
-          raise("Test case merger does not exist for adapter_class: #{adapter_class}")
-        Kernel.const_get(merger_class).new(test_files).call
-      end
-
-      def initialize(test_files)
-        @test_files = test_files
-      end
-
-      def call
-        raise NotImplementedError
-      end
-
-      private
-
-      attr_reader :test_files
-    end
-  end
-end

data/lib/knapsack_pro/test_files_with_test_cases_composer.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module KnapsackPro
-  class TestFilesWithTestCasesComposer
-    # Args:
-    #   All 3 arguments have structure: [{ 'path' => 'spec/a_spec.rb', 'time_execution' => 0.0 }]
-    #   time_execution is not always present (but it's not relevant here)
-    #
-    #   test_files - list of test files that you want to run tests for
-    #   slow_test_files - list of slow test files that should be split by test cases
-    #   test_file_cases - list of paths to test cases (test examples) inside of all slow test files (slow_test_files)
-    # Return:
-    #   Test files and test cases paths (it excludes test files that should be split by test cases (test examples))
-    def self.call(test_files, slow_test_files, test_file_cases)
-      slow_test_file_paths = KnapsackPro::TestFilePresenter.paths(slow_test_files)
-
-      test_files_without_slow_test_files = test_files.reject do |test_file|
-        slow_test_file_paths.include?(test_file.fetch('path'))
-      end
-
-      test_files_without_slow_test_files + test_file_cases
-    end
-  end
-end