knapsack_pro 1.20.2 → 1.22.3

data/lib/knapsack_pro/client/connection.rb

@@ -4,8 +4,8 @@ module KnapsackPro
       class ServerError < StandardError; end
 
       TIMEOUT = 15
-      MAX_RETRY = 3
-      REQUEST_RETRY_TIMEBOX = 4
+      MAX_RETRY = -> { KnapsackPro::Config::Env.fallback_mode_enabled? ? 3 : 6 }
+      REQUEST_RETRY_TIMEBOX = 8
 
       def initialize(action)
         @action = action
@@ -48,11 +48,7 @@ module KnapsackPro
       end
 
       def request_hash
-        action
-        .request_hash
-        .merge({
-          test_suite_token: test_suite_token
-        })
+        action.request_hash
       end
 
       def request_body
@@ -69,6 +65,7 @@ module KnapsackPro
           'Accept' => 'application/json',
           'KNAPSACK-PRO-CLIENT-NAME' => client_name,
           'KNAPSACK-PRO-CLIENT-VERSION' => KnapsackPro::VERSION,
+          'KNAPSACK-PRO-TEST-SUITE-TOKEN' => test_suite_token,
         }
       end
 
@@ -95,19 +92,15 @@ module KnapsackPro
         !seed.nil?
       end
 
-      def post
+      def make_request(&block)
         retries ||= 0
-        uri = URI.parse(endpoint_url)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = (uri.scheme == 'https')
-        http.open_timeout = TIMEOUT
-        http.read_timeout = TIMEOUT
 
-        @http_response = http.post(uri.path, request_body, json_headers)
+        @http_response = block.call
         @response_body = parse_response_body(http_response.body)
 
         request_uuid = http_response.header['X-Request-Id'] || 'N/A'
 
+        logger.debug("#{action.http_method.to_s.upcase} #{endpoint_url}")
         logger.debug("API request UUID: #{request_uuid}")
         logger.debug("Test suite split seed: #{seed}") if has_seed?
         logger.debug('API response:')
@@ -125,15 +118,44 @@ module KnapsackPro
       rescue ServerError, Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::EPIPE, EOFError, SocketError, Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError => e
         logger.warn(e.inspect)
         retries += 1
-        if retries < MAX_RETRY
+        if retries < MAX_RETRY.call
           wait = retries * REQUEST_RETRY_TIMEBOX
           logger.warn("Wait #{wait}s and retry request to Knapsack Pro API.")
-          Kernel.sleep(wait)
+          print_every = 2 # seconds
+          (wait / print_every).ceil.times do |i|
+            logger.warn("Next request in #{wait - i * print_every}s...")
+            Kernel.sleep(print_every)
+          end
           retry
         else
           response_body
         end
       end
+
+      def build_http(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == 'https')
+        http.open_timeout = TIMEOUT
+        http.read_timeout = TIMEOUT
+        http
+      end
+
+      def post
+        uri = URI.parse(endpoint_url)
+        http = build_http(uri)
+        make_request do
+          http.post(uri.path, request_body, json_headers)
+        end
+      end
+
+      def get
+        uri = URI.parse(endpoint_url)
+        uri.query = URI.encode_www_form(request_hash)
+        http = build_http(uri)
+        make_request do
+          http.get(uri, json_headers)
+        end
+      end
     end
   end
 end

data/lib/knapsack_pro/config/env.rb

@@ -55,6 +55,10 @@ module KnapsackPro
           ENV['KNAPSACK_PRO_TEST_FILE_PATTERN']
         end
 
+        def slow_test_file_pattern
+          ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN']
+        end
+
         def test_file_exclude_pattern
           ENV['KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN']
         end

data/lib/knapsack_pro/queue_allocator.rb

@@ -1,7 +1,8 @@
 module KnapsackPro
   class QueueAllocator
     def initialize(args)
-      @test_files = args.fetch(:test_files)
+      @fast_and_slow_test_files_to_run = args.fetch(:fast_and_slow_test_files_to_run)
+      @fallback_mode_test_files = args.fetch(:fallback_mode_test_files)
       @ci_node_total = args.fetch(:ci_node_total)
       @ci_node_index = args.fetch(:ci_node_index)
       @ci_node_build_id = args.fetch(:ci_node_build_id)
@@ -36,14 +37,15 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_files,
+    attr_reader :fast_and_slow_test_files_to_run,
+      :fallback_mode_test_files,
       :ci_node_total,
       :ci_node_index,
       :ci_node_build_id,
       :repository_adapter
 
     def encrypted_test_files
-      KnapsackPro::Crypto::Encryptor.call(test_files)
+      KnapsackPro::Crypto::Encryptor.call(fast_and_slow_test_files_to_run)
     end
 
     def encrypted_branch
@@ -63,12 +65,12 @@ module KnapsackPro
     end
 
     def prepare_test_files(response)
-      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(test_files, response['test_files'])
+      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(fast_and_slow_test_files_to_run, response['test_files'])
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
 
     def fallback_test_files(executed_test_files)
-      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_files, ci_node_total)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(fallback_mode_test_files, ci_node_total)
       test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
       KnapsackPro::TestFilePresenter.paths(test_files_for_node_index) - executed_test_files
     end

data/lib/knapsack_pro/queue_allocator_builder.rb

@@ -2,7 +2,8 @@ module KnapsackPro
   class QueueAllocatorBuilder < BaseAllocatorBuilder
     def allocator
       KnapsackPro::QueueAllocator.new(
-        test_files: test_files,
+        fast_and_slow_test_files_to_run: fast_and_slow_test_files_to_run,
+        fallback_mode_test_files: fallback_mode_test_files,
         ci_node_total: env.ci_node_total,
         ci_node_index: env.ci_node_index,
         ci_node_build_id: env.ci_node_build_id,

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -16,7 +16,7 @@ module KnapsackPro
           cli_args = (args || '').split
           # if user didn't provide the format then use explicitly default progress formatter
           # in order to avoid KnapsackPro::Formatters::RSpecQueueSummaryFormatter being the only default formatter
-          if !cli_args.include?('--format') && !cli_args.include?('-f')
+          if !cli_args.any? { |arg| arg.start_with?('-f') || arg.start_with?('--format')}
             cli_args += ['--format', 'progress']
           end
           cli_args += [
@@ -134,6 +134,13 @@ module KnapsackPro
           RSpec.world.example_groups.clear
           RSpec.configuration.start_time = ::RSpec::Core::Time.now
 
+          if KnapsackPro::Config::Env.rspec_split_by_test_examples?
+            # Reset example group counts to ensure scoped example ids in metadata
+            # have correct index (not increased by each subsequent run).
+            # Solves this problem: https://github.com/rspec/rspec-core/issues/2721
+            RSpec.world.instance_variable_set(:@example_group_counts_by_spec_file, Hash.new(0))
+          end
+
           # skip reset filters for old RSpec versions
           if RSpec.configuration.respond_to?(:reset_filters)
             RSpec.configuration.reset_filters

data/lib/knapsack_pro/runners/rspec_runner.rb

@@ -16,8 +16,11 @@ module KnapsackPro
           end
 
           RSpec::Core::RakeTask.new(task_name) do |t|
-            t.rspec_opts = "#{args} --default-path #{runner.test_dir}"
-            t.pattern = runner.test_file_paths
+            # we cannot pass runner.test_file_paths array to t.pattern
+            # because pattern does not accept test example path like spec/a_spec.rb[1:2]
+            # instead we pass test files and test example paths to t.rspec_opts
+            t.pattern = []
+            t.rspec_opts = "#{args} --default-path #{runner.test_dir} #{runner.stringify_test_file_paths}"
           end
           Rake::Task[task_name].invoke
         end

data/lib/knapsack_pro/slow_test_file_determiner.rb

@@ -0,0 +1,33 @@
+module KnapsackPro
+  class SlowTestFileDeterminer
+    TIME_THRESHOLD_PER_CI_NODE = 0.7 # 70%
+    REPORT_DIR = 'tmp/knapsack_pro/slow_test_file_determiner'
+
+    # test_files: { 'path' => 'a_spec.rb', 'time_execution' => 0.0 }
+    # time_execution: of build distribution (total time of CI build run)
+    def self.call(test_files, time_execution)
+      time_threshold = (time_execution / KnapsackPro::Config::Env.ci_node_total) * TIME_THRESHOLD_PER_CI_NODE
+
+      test_files.select do |test_file|
+        test_file.fetch('time_execution') >= time_threshold
+      end
+    end
+
+    def self.save_to_json_report(test_files)
+      FileUtils.mkdir_p(REPORT_DIR)
+      File.write(report_path, test_files.to_json)
+    end
+
+    def self.read_from_json_report
+      raise 'Report with slow test files was not generated yet. If you have enabled split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases and you see this error it means that your tests accidentally cleaned up tmp/knapsack_pro directory. Please do not remove this directory during tests runtime!' unless File.exists?(report_path)
+      slow_test_files_json_report = File.read(report_path)
+      JSON.parse(slow_test_files_json_report)
+    end
+
+    private
+
+    def self.report_path
+      "#{REPORT_DIR}/slow_test_files_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+    end
+  end
+end

data/lib/knapsack_pro/slow_test_file_finder.rb

@@ -0,0 +1,27 @@
+module KnapsackPro
+  class SlowTestFileFinder
+    # Get recorded test files from API.
+    # Find slow tests among them that are still present on the disk.
+    # Save slow test files in json file on the disk.
+    # Returns slow test files.
+    def self.call(adapter_class)
+      if KnapsackPro::Config::Env.test_files_encrypted?
+        raise 'Split by test cases is not possible when you have enabled test file names encryption ( https://github.com/KnapsackPro/knapsack_pro-ruby#test-file-names-encryption ). You need to disable encryption with KNAPSACK_PRO_TEST_FILES_ENCRYPTED=false in order to use split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases'
+      end
+
+      # get list of recorded test files for last CI Build
+      build_distribution_entity = KnapsackPro::BuildDistributionFetcher.call
+      test_files_from_api = build_distribution_entity.test_files
+
+      merged_test_files_from_api = KnapsackPro::TestCaseMergers::BaseMerger.call(adapter_class, test_files_from_api)
+
+      test_files_existing_on_disk = KnapsackPro::TestFileFinder.select_test_files_that_can_be_run(adapter_class, merged_test_files_from_api)
+
+      slow_test_files = KnapsackPro::SlowTestFileDeterminer.call(test_files_existing_on_disk, build_distribution_entity.time_execution)
+
+      KnapsackPro::SlowTestFileDeterminer.save_to_json_report(slow_test_files)
+
+      slow_test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_detectors/rspec_test_example_detector.rb

@@ -2,7 +2,6 @@ module KnapsackPro
   module TestCaseDetectors
     class RSpecTestExampleDetector
       REPORT_DIR = 'tmp/knapsack_pro/test_case_detectors/rspec'
-      REPORT_PATH = "#{REPORT_DIR}/rspec_dry_run_json_report.json"
 
       def generate_json_report
         require 'rspec/core'
@@ -18,13 +17,19 @@ module KnapsackPro
         ensure_report_dir_exists
         remove_old_json_report
 
-        test_file_paths = KnapsackPro::TestFileFinder.call(test_file_pattern)
+        test_file_entities = slow_test_files
+
+        if test_file_entities.empty?
+          no_examples_json = { examples: [] }.to_json
+          File.write(report_path, no_examples_json)
+          return
+        end
 
         cli_args = cli_format + [
           '--dry-run',
-          '--out', REPORT_PATH,
+          '--out', report_path,
           '--default-path', test_dir,
-        ] + test_file_paths.map { |t| t.fetch('path') }
+        ] + KnapsackPro::TestFilePresenter.paths(test_file_entities)
         options = RSpec::Core::ConfigurationOptions.new(cli_args)
         exit_code = RSpec::Core::Runner.new(options).run($stderr, $stdout)
         if exit_code != 0
@@ -33,9 +38,9 @@ module KnapsackPro
       end
 
       def test_file_example_paths
-        raise "No report found at #{REPORT_PATH}" unless File.exists?(REPORT_PATH)
+        raise "No report found at #{report_path}" unless File.exists?(report_path)
 
-        json_report = File.read(REPORT_PATH)
+        json_report = File.read(report_path)
         hash_report = JSON.parse(json_report)
         hash_report
           .fetch('examples')
@@ -43,8 +48,22 @@ module KnapsackPro
           .map { |path_with_example_id| test_file_hash_for(path_with_example_id) }
       end
 
+      def slow_test_files
+        if KnapsackPro::Config::Env.slow_test_file_pattern
+          KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+        else
+          # read slow test files from JSON file on disk that was generated
+          # by lib/knapsack_pro/base_allocator_builder.rb
+          KnapsackPro::SlowTestFileDeterminer.read_from_json_report
+        end
+      end
+
       private
 
+      def report_path
+        "#{REPORT_DIR}/rspec_dry_run_json_report_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+      end
+
       def adapter_class
         KnapsackPro::Adapters::RSpecAdapter
       end
@@ -62,7 +81,7 @@ module KnapsackPro
       end
 
       def remove_old_json_report
-        File.delete(REPORT_PATH) if File.exists?(REPORT_PATH)
+        File.delete(report_path) if File.exists?(report_path)
       end
 
       def test_file_hash_for(test_file_path)

data/lib/knapsack_pro/test_case_mergers/base_merger.rb

@@ -0,0 +1,29 @@
+module KnapsackPro
+  module TestCaseMergers
+    class BaseMerger
+      # values must be string to avoid circular dependency problem during loading files
+      ADAPTER_TO_MERGER_MAP = {
+        KnapsackPro::Adapters::RSpecAdapter => 'KnapsackPro::TestCaseMergers::RSpecMerger',
+      }
+
+      def self.call(adapter_class, test_files)
+        merger_class =
+          ADAPTER_TO_MERGER_MAP[adapter_class] ||
+          raise("Test case merger does not exist for adapter_class: #{adapter_class}")
+        Kernel.const_get(merger_class).new(test_files).call
+      end
+
+      def initialize(test_files)
+        @test_files = test_files
+      end
+
+      def call
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_reader :test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_mergers/rspec_merger.rb

@@ -0,0 +1,34 @@
+module KnapsackPro
+  module TestCaseMergers
+    class RSpecMerger < BaseMerger
+      def call
+        merged_test_files_hash = {}
+        test_files.each do |test_file|
+          test_file_path = extract_test_file_path(test_file.fetch('path'))
+
+          # must be float (default type for time execution from API)
+          merged_test_files_hash[test_file_path] ||= 0.0
+          merged_test_files_hash[test_file_path] += test_file.fetch('time_execution')
+        end
+
+        merged_test_files = []
+        merged_test_files_hash.each do |path, time_execution|
+          merged_test_files << {
+            'path' => path,
+            'time_execution' => time_execution
+          }
+        end
+        merged_test_files
+      end
+
+      private
+
+      # path - can be:
+      # test file path: spec/a_spec.rb
+      # or test example path: spec/a_spec.rb[1:1]
+      def extract_test_file_path(path)
+        path.gsub(/\.rb\[.+\]$/, '.rb')
+      end
+    end
+  end
+end

data/lib/knapsack_pro/test_file_finder.rb

@@ -1,11 +1,49 @@
 module KnapsackPro
   class TestFileFinder
-    def self.call(test_file_pattern)
-      new(test_file_pattern).call
+    def self.call(test_file_pattern, test_file_list_enabled: true)
+      new(test_file_pattern, test_file_list_enabled).call
     end
 
-    def initialize(test_file_pattern)
+    # finds slow test files on disk based on ENV patterns
+    # returns example: [{ 'path' => 'a_spec.rb' }]
+    def self.slow_test_files_by_pattern(adapter_class)
+      raise 'KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN is not defined' unless KnapsackPro::Config::Env.slow_test_file_pattern
+
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      slow_test_file_entities = call(KnapsackPro::Config::Env.slow_test_file_pattern, test_file_list_enabled: false)
+
+      # slow test files (KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN)
+      # should be subset of test file pattern (KNAPSACK_PRO_TEST_FILE_PATTERN)
+      slow_test_file_entities & test_file_entities
+    end
+
+    # Args:
+    #   test_file_entities_to_run - it can be list of slow test files that you want to run
+    # Return:
+    #   subset of test_file_entities_to_run that are present on disk and it is subset of tests matching pattern KNAPSACK_PRO_TEST_FILE_PATTERN
+    #   Thanks to that we can select only slow test files that are within list of test file pattern we want to run tests for
+    def self.select_test_files_that_can_be_run(adapter_class, test_file_entities_to_run)
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      test_file_paths_existing_on_disk = KnapsackPro::TestFilePresenter.paths(test_file_entities)
+
+      selected_test_files = []
+
+      test_file_entities_to_run.each do |test_file_entity|
+        if test_file_paths_existing_on_disk.include?(test_file_entity.fetch('path'))
+          selected_test_files << test_file_entity
+        end
+      end
+
+      selected_test_files
+    end
+
+    def initialize(test_file_pattern, test_file_list_enabled)
       @test_file_pattern = test_file_pattern
+      @test_file_list_enabled = test_file_list_enabled
     end
 
     def call
@@ -18,10 +56,10 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_file_pattern
+    attr_reader :test_file_pattern, :test_file_list_enabled
 
     def test_files
-      if KnapsackPro::Config::Env.test_file_list
+      if test_file_list_enabled && KnapsackPro::Config::Env.test_file_list
         return KnapsackPro::Config::Env.test_file_list.split(',').map(&:strip)
       end