knapsack_pro 1.20.2 → 1.21.0

data/lib/knapsack_pro/queue_allocator.rb

@@ -1,7 +1,8 @@
 module KnapsackPro
   class QueueAllocator
     def initialize(args)
-      @test_files = args.fetch(:test_files)
+      @fast_and_slow_test_files_to_run = args.fetch(:fast_and_slow_test_files_to_run)
+      @fallback_mode_test_files = args.fetch(:fallback_mode_test_files)
       @ci_node_total = args.fetch(:ci_node_total)
       @ci_node_index = args.fetch(:ci_node_index)
       @ci_node_build_id = args.fetch(:ci_node_build_id)
@@ -36,14 +37,15 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_files,
+    attr_reader :fast_and_slow_test_files_to_run,
+      :fallback_mode_test_files,
       :ci_node_total,
       :ci_node_index,
       :ci_node_build_id,
       :repository_adapter
 
     def encrypted_test_files
-      KnapsackPro::Crypto::Encryptor.call(test_files)
+      KnapsackPro::Crypto::Encryptor.call(fast_and_slow_test_files_to_run)
     end
 
     def encrypted_branch
@@ -63,12 +65,12 @@ module KnapsackPro
     end
 
     def prepare_test_files(response)
-      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(test_files, response['test_files'])
+      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(fast_and_slow_test_files_to_run, response['test_files'])
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
 
     def fallback_test_files(executed_test_files)
-      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_files, ci_node_total)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(fallback_mode_test_files, ci_node_total)
       test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
       KnapsackPro::TestFilePresenter.paths(test_files_for_node_index) - executed_test_files
     end

data/lib/knapsack_pro/queue_allocator_builder.rb

@@ -2,7 +2,8 @@ module KnapsackPro
   class QueueAllocatorBuilder < BaseAllocatorBuilder
     def allocator
       KnapsackPro::QueueAllocator.new(
-        test_files: test_files,
+        fast_and_slow_test_files_to_run: fast_and_slow_test_files_to_run,
+        fallback_mode_test_files: fallback_mode_test_files,
         ci_node_total: env.ci_node_total,
         ci_node_index: env.ci_node_index,
         ci_node_build_id: env.ci_node_build_id,

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -134,6 +134,13 @@ module KnapsackPro
           RSpec.world.example_groups.clear
           RSpec.configuration.start_time = ::RSpec::Core::Time.now
 
+          if KnapsackPro::Config::Env.rspec_split_by_test_examples?
+            # Reset example group counts to ensure scoped example ids in metadata
+            # have correct index (not increased by each subsequent run).
+            # Solves this problem: https://github.com/rspec/rspec-core/issues/2721
+            RSpec.world.instance_variable_set(:@example_group_counts_by_spec_file, Hash.new(0))
+          end
+
           # skip reset filters for old RSpec versions
           if RSpec.configuration.respond_to?(:reset_filters)
             RSpec.configuration.reset_filters

data/lib/knapsack_pro/runners/rspec_runner.rb

@@ -16,8 +16,11 @@ module KnapsackPro
           end
 
           RSpec::Core::RakeTask.new(task_name) do |t|
-            t.rspec_opts = "#{args} --default-path #{runner.test_dir}"
-            t.pattern = runner.test_file_paths
+            # we cannot pass runner.test_file_paths array to t.pattern
+            # because pattern does not accept test example path like spec/a_spec.rb[1:2]
+            # instead we pass test files and test example paths to t.rspec_opts
+            t.pattern = []
+            t.rspec_opts = "#{args} --default-path #{runner.test_dir} #{runner.stringify_test_file_paths}"
           end
           Rake::Task[task_name].invoke
         end

data/lib/knapsack_pro/slow_test_file_determiner.rb

@@ -0,0 +1,28 @@
+module KnapsackPro
+  class SlowTestFileDeterminer
+    TIME_THRESHOLD_PER_CI_NODE = 0.7 # 70%
+    REPORT_DIR = 'tmp/knapsack_pro/slow_test_file_determiner'
+    REPORT_PATH = "#{REPORT_DIR}/slow_test_files.json"
+
+    # test_files: { 'path' => 'a_spec.rb', 'time_execution' => 0.0 }
+    # time_execution: of build distribution (total time of CI build run)
+    def self.call(test_files, time_execution)
+      time_threshold = (time_execution / KnapsackPro::Config::Env.ci_node_total) * TIME_THRESHOLD_PER_CI_NODE
+
+      test_files.select do |test_file|
+        test_file.fetch('time_execution') >= time_threshold
+      end
+    end
+
+    def self.save_to_json_report(test_files)
+      FileUtils.mkdir_p(REPORT_DIR)
+      File.write(REPORT_PATH, test_files.to_json)
+    end
+
+    def self.read_from_json_report
+      raise 'Report with slow test files was not generated yet. If you have enabled split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases and you see this error it means that your tests accidentally cleaned up tmp/knapsack_pro directory. Please do not remove this directory during tests runtime!' unless File.exists?(REPORT_PATH)
+      slow_test_files_json_report = File.read(REPORT_PATH)
+      JSON.parse(slow_test_files_json_report)
+    end
+  end
+end

data/lib/knapsack_pro/slow_test_file_finder.rb

@@ -0,0 +1,27 @@
+module KnapsackPro
+  class SlowTestFileFinder
+    # Get recorded test files from API.
+    # Find slow tests among them that are still present on the disk.
+    # Save slow test files in json file on the disk.
+    # Returns slow test files.
+    def self.call(adapter_class)
+      if KnapsackPro::Config::Env.test_files_encrypted?
+        raise 'Split by test cases is not possible when you have enabled test file names encryption ( https://github.com/KnapsackPro/knapsack_pro-ruby#test-file-names-encryption ). You need to disable encryption with KNAPSACK_PRO_TEST_FILES_ENCRYPTED=false in order to use split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases'
+      end
+
+      # get list of recorded test files for last CI Build
+      build_distribution_entity = KnapsackPro::BuildDistributionFetcher.call
+      test_files_from_api = build_distribution_entity.test_files
+
+      merged_test_files_from_api = KnapsackPro::TestCaseMergers::BaseMerger.call(adapter_class, test_files_from_api)
+
+      test_files_existing_on_disk = KnapsackPro::TestFileFinder.select_test_files_that_can_be_run(adapter_class, merged_test_files_from_api)
+
+      slow_test_files = KnapsackPro::SlowTestFileDeterminer.call(test_files_existing_on_disk, build_distribution_entity.time_execution)
+
+      KnapsackPro::SlowTestFileDeterminer.save_to_json_report(slow_test_files)
+
+      slow_test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_detectors/rspec_test_example_detector.rb

@@ -18,13 +18,19 @@ module KnapsackPro
         ensure_report_dir_exists
         remove_old_json_report
 
-        test_file_paths = KnapsackPro::TestFileFinder.call(test_file_pattern)
+        test_file_entities = slow_test_files
+
+        if test_file_entities.empty?
+          no_examples_json = { examples: [] }.to_json
+          File.write(REPORT_PATH, no_examples_json)
+          return
+        end
 
         cli_args = cli_format + [
           '--dry-run',
           '--out', REPORT_PATH,
           '--default-path', test_dir,
-        ] + test_file_paths.map { |t| t.fetch('path') }
+        ] + KnapsackPro::TestFilePresenter.paths(test_file_entities)
         options = RSpec::Core::ConfigurationOptions.new(cli_args)
         exit_code = RSpec::Core::Runner.new(options).run($stderr, $stdout)
         if exit_code != 0
@@ -43,6 +49,16 @@ module KnapsackPro
           .map { |path_with_example_id| test_file_hash_for(path_with_example_id) }
       end
 
+      def slow_test_files
+        if KnapsackPro::Config::Env.slow_test_file_pattern
+          KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+        else
+          # read slow test files from JSON file on disk that was generated
+          # by lib/knapsack_pro/base_allocator_builder.rb
+          KnapsackPro::SlowTestFileDeterminer.read_from_json_report
+        end
+      end
+
       private
 
       def adapter_class

data/lib/knapsack_pro/test_case_mergers/base_merger.rb

@@ -0,0 +1,29 @@
+module KnapsackPro
+  module TestCaseMergers
+    class BaseMerger
+      # values must be string to avoid circular dependency problem during loading files
+      ADAPTER_TO_MERGER_MAP = {
+        KnapsackPro::Adapters::RSpecAdapter => 'KnapsackPro::TestCaseMergers::RSpecMerger',
+      }
+
+      def self.call(adapter_class, test_files)
+        merger_class =
+          ADAPTER_TO_MERGER_MAP[adapter_class] ||
+          raise("Test case merger does not exist for adapter_class: #{adapter_class}")
+        Kernel.const_get(merger_class).new(test_files).call
+      end
+
+      def initialize(test_files)
+        @test_files = test_files
+      end
+
+      def call
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_reader :test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_mergers/rspec_merger.rb

@@ -0,0 +1,34 @@
+module KnapsackPro
+  module TestCaseMergers
+    class RSpecMerger < BaseMerger
+      def call
+        merged_test_files_hash = {}
+        test_files.each do |test_file|
+          test_file_path = extract_test_file_path(test_file.fetch('path'))
+
+          # must be float (default type for time execution from API)
+          merged_test_files_hash[test_file_path] ||= 0.0
+          merged_test_files_hash[test_file_path] += test_file.fetch('time_execution')
+        end
+
+        merged_test_files = []
+        merged_test_files_hash.each do |path, time_execution|
+          merged_test_files << {
+            'path' => path,
+            'time_execution' => time_execution
+          }
+        end
+        merged_test_files
+      end
+
+      private
+
+      # path - can be:
+      # test file path: spec/a_spec.rb
+      # or test example path: spec/a_spec.rb[1:1]
+      def extract_test_file_path(path)
+        path.gsub(/\.rb\[.+\]$/, '.rb')
+      end
+    end
+  end
+end

data/lib/knapsack_pro/test_file_finder.rb

@@ -1,11 +1,49 @@
 module KnapsackPro
   class TestFileFinder
-    def self.call(test_file_pattern)
-      new(test_file_pattern).call
+    def self.call(test_file_pattern, test_file_list_enabled: true)
+      new(test_file_pattern, test_file_list_enabled).call
     end
 
-    def initialize(test_file_pattern)
+    # finds slow test files on disk based on ENV patterns
+    # returns example: [{ 'path' => 'a_spec.rb' }]
+    def self.slow_test_files_by_pattern(adapter_class)
+      raise 'KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN is not defined' unless KnapsackPro::Config::Env.slow_test_file_pattern
+
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      slow_test_file_entities = call(KnapsackPro::Config::Env.slow_test_file_pattern, test_file_list_enabled: false)
+
+      # slow test files (KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN)
+      # should be subset of test file pattern (KNAPSACK_PRO_TEST_FILE_PATTERN)
+      slow_test_file_entities & test_file_entities
+    end
+
+    # Args:
+    #   test_file_entities_to_run - it can be list of slow test files that you want to run
+    # Return:
+    #   subset of test_file_entities_to_run that are present on disk and it is subset of tests matching pattern KNAPSACK_PRO_TEST_FILE_PATTERN
+    #   Thanks to that we can select only slow test files that are within list of test file pattern we want to run tests for
+    def self.select_test_files_that_can_be_run(adapter_class, test_file_entities_to_run)
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      test_file_paths_existing_on_disk = KnapsackPro::TestFilePresenter.paths(test_file_entities)
+
+      selected_test_files = []
+
+      test_file_entities_to_run.each do |test_file_entity|
+        if test_file_paths_existing_on_disk.include?(test_file_entity.fetch('path'))
+          selected_test_files << test_file_entity
+        end
+      end
+
+      selected_test_files
+    end
+
+    def initialize(test_file_pattern, test_file_list_enabled)
       @test_file_pattern = test_file_pattern
+      @test_file_list_enabled = test_file_list_enabled
     end
 
     def call
@@ -18,10 +56,10 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_file_pattern
+    attr_reader :test_file_pattern, :test_file_list_enabled
 
     def test_files
-      if KnapsackPro::Config::Env.test_file_list
+      if test_file_list_enabled && KnapsackPro::Config::Env.test_file_list
         return KnapsackPro::Config::Env.test_file_list.split(',').map(&:strip)
       end
 

data/lib/knapsack_pro/test_files_with_test_cases_composer.rb

@@ -0,0 +1,22 @@
+module KnapsackPro
+  class TestFilesWithTestCasesComposer
+    # Args:
+    #   All 3 arguments have structure: [{ 'path' => 'spec/a_spec.rb', 'time_execution' => 0.0 }]
+    #   time_execution is not always present (but it's not relevant here)
+    #
+    #   test_files - list of test files that you want to run tests for
+    #   slow_test_files - list of slow test files that should be split by test cases
+    #   test_file_cases - list of paths to test cases (test examples) inside of all slow test files (slow_test_files)
+    # Return:
+    #   Test files and test cases paths (it excludes test files that should be split by test cases (test examples))
+    def self.call(test_files, slow_test_files, test_file_cases)
+      slow_test_file_paths = KnapsackPro::TestFilePresenter.paths(slow_test_files)
+
+      test_files_without_slow_test_files = test_files.reject do |test_file|
+        slow_test_file_paths.include?(test_file.fetch('path'))
+      end
+
+      test_files_without_slow_test_files + test_file_cases
+    end
+  end
+end

data/lib/knapsack_pro/version.rb

@@ -1,3 +1,3 @@
 module KnapsackPro
-  VERSION = '1.20.2'
+  VERSION = '1.21.0'
 end

data/spec/knapsack_pro/adapters/base_adapter_spec.rb

@@ -3,6 +3,61 @@ describe KnapsackPro::Adapters::BaseAdapter do
     expect(described_class::TEST_DIR_PATTERN).to eq 'test/**{,/*/**}/*_test.rb'
   end
 
+  shared_examples '.slow_test_file? method' do
+    context 'when test_file_path is in slow test file paths' do
+      # add ./ before path to ensure KnapsackPro::TestFileCleaner.clean will clean it
+      let(:test_file_path) { './spec/models/user_spec.rb' }
+
+      it do
+        expect(subject).to be true
+      end
+    end
+
+    context 'when test_file_path is not in slow test file paths' do
+      let(:test_file_path) { './spec/models/article_spec.rb' }
+
+      it do
+        expect(subject).to be false
+      end
+    end
+  end
+
+  describe '.slow_test_file?' do
+    let(:adapter_class) { double }
+    let(:slow_test_files) do
+      [
+        { 'path' => 'spec/models/user_spec.rb' },
+        { 'path' => 'spec/controllers/users_spec.rb' },
+      ]
+    end
+
+    subject { described_class.slow_test_file?(adapter_class, test_file_path) }
+
+    before do
+      # reset class variable
+      described_class.instance_variable_set(:@slow_test_file_paths, nil)
+    end
+
+    context 'when slow test file pattern is present' do
+      before do
+        stub_const('ENV', {
+          'KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN' => '{spec/models/*_spec.rb}',
+        })
+        expect(KnapsackPro::TestFileFinder).to receive(:slow_test_files_by_pattern).with(adapter_class).and_return(slow_test_files)
+      end
+
+      it_behaves_like '.slow_test_file? method'
+    end
+
+    context 'when slow test file pattern is not present' do
+      before do
+        expect(KnapsackPro::SlowTestFileDeterminer).to receive(:read_from_json_report).and_return(slow_test_files)
+      end
+
+      it_behaves_like '.slow_test_file? method'
+    end
+  end
+
   describe '.bind' do
     let(:adapter) { instance_double(described_class) }
 

data/spec/knapsack_pro/adapters/rspec_adapter_spec.rb

@@ -71,22 +71,21 @@ describe KnapsackPro::Adapters::RSpecAdapter do
       let(:tracker) { instance_double(KnapsackPro::Tracker) }
       let(:logger) { instance_double(Logger) }
       let(:global_time) { 'Global time: 01m 05s' }
+      let(:test_path) { 'spec/a_spec.rb' }
+      let(:example) { double }
+      let(:example_group) { double }
+      let(:current_example) do
+        OpenStruct.new(metadata: {
+          example_group: example_group
+        })
+      end
 
       context 'when rspec split by test examples is disabled' do
-        let(:test_path) { 'spec/a_spec.rb' }
-        let(:example) { double }
-        let(:example_group) { double }
-        let(:current_example) do
-          OpenStruct.new(metadata: {
-            example_group: example_group
-          })
-        end
-
         before do
           expect(KnapsackPro::Config::Env).to receive(:rspec_split_by_test_examples?).and_return(false)
         end
 
-        it do
+        it 'records time for current test path' do
           expect(config).to receive(:around).with(:each).and_yield(example)
           expect(config).to receive(:after).with(:suite).and_yield
           expect(::RSpec).to receive(:configure).and_yield(config)
@@ -112,32 +111,69 @@ describe KnapsackPro::Adapters::RSpecAdapter do
 
       context 'when rspec split by test examples is enabled' do
         let(:test_example_path) { 'spec/a_spec.rb[1:1]' }
-        let(:example) { double }
 
         before do
           expect(KnapsackPro::Config::Env).to receive(:rspec_split_by_test_examples?).and_return(true)
         end
 
-        it do
-          expect(config).to receive(:around).with(:each).and_yield(example)
-          expect(config).to receive(:after).with(:suite).and_yield
-          expect(::RSpec).to receive(:configure).and_yield(config)
+        context 'when current test_path is a slow test file' do
+          before do
+            expect(described_class).to receive(:slow_test_file?).with(described_class, test_path).and_return(true)
+          end
 
-          expect(example).to receive(:id).and_return(test_example_path)
+          it 'records time for example.id' do
+            expect(example).to receive(:id).and_return(test_example_path)
 
-          allow(KnapsackPro).to receive(:tracker).and_return(tracker)
-          expect(tracker).to receive(:current_test_path=).with(test_example_path)
-          expect(tracker).to receive(:start_timer)
+            expect(config).to receive(:around).with(:each).and_yield(example)
+            expect(config).to receive(:after).with(:suite).and_yield
+            expect(::RSpec).to receive(:configure).and_yield(config)
 
-          expect(example).to receive(:run)
+            expect(::RSpec).to receive(:current_example).twice.and_return(current_example)
+            expect(described_class).to receive(:test_path).with(example_group).and_return(test_path)
 
-          expect(tracker).to receive(:stop_timer)
+            allow(KnapsackPro).to receive(:tracker).and_return(tracker)
+            expect(tracker).to receive(:current_test_path=).with(test_example_path)
+            expect(tracker).to receive(:start_timer)
 
-          expect(KnapsackPro::Presenter).to receive(:global_time).and_return(global_time)
-          expect(KnapsackPro).to receive(:logger).and_return(logger)
-          expect(logger).to receive(:debug).with(global_time)
+            expect(example).to receive(:run)
 
-          subject.bind_time_tracker
+            expect(tracker).to receive(:stop_timer)
+
+            expect(KnapsackPro::Presenter).to receive(:global_time).and_return(global_time)
+            expect(KnapsackPro).to receive(:logger).and_return(logger)
+            expect(logger).to receive(:debug).with(global_time)
+
+            subject.bind_time_tracker
+          end
+        end
+
+        context 'when current test_path is not a slow test file' do
+          before do
+            expect(described_class).to receive(:slow_test_file?).with(described_class, test_path).and_return(false)
+          end
+
+          it 'records time for current test path' do
+            expect(config).to receive(:around).with(:each).and_yield(example)
+            expect(config).to receive(:after).with(:suite).and_yield
+            expect(::RSpec).to receive(:configure).and_yield(config)
+
+            expect(::RSpec).to receive(:current_example).twice.and_return(current_example)
+            expect(described_class).to receive(:test_path).with(example_group).and_return(test_path)
+
+            allow(KnapsackPro).to receive(:tracker).and_return(tracker)
+            expect(tracker).to receive(:current_test_path=).with(test_path)
+            expect(tracker).to receive(:start_timer)
+
+            expect(example).to receive(:run)
+
+            expect(tracker).to receive(:stop_timer)
+
+            expect(KnapsackPro::Presenter).to receive(:global_time).and_return(global_time)
+            expect(KnapsackPro).to receive(:logger).and_return(logger)
+            expect(logger).to receive(:debug).with(global_time)
+
+            subject.bind_time_tracker
+          end
         end
       end
     end