knapsack_pro 1.20.0 → 1.22.1

data/lib/knapsack_pro/client/connection.rb

@@ -4,8 +4,8 @@ module KnapsackPro
       class ServerError < StandardError; end
 
       TIMEOUT = 15
-      MAX_RETRY = 3
-      REQUEST_RETRY_TIMEBOX = 4
+      MAX_RETRY = -> { KnapsackPro::Config::Env.fallback_mode_enabled? ? 3 : 6 }
+      REQUEST_RETRY_TIMEBOX = 8
 
       def initialize(action)
         @action = action
@@ -48,11 +48,7 @@ module KnapsackPro
       end
 
       def request_hash
-        action
-        .request_hash
-        .merge({
-          test_suite_token: test_suite_token
-        })
+        action.request_hash
       end
 
       def request_body
@@ -69,6 +65,7 @@ module KnapsackPro
           'Accept' => 'application/json',
           'KNAPSACK-PRO-CLIENT-NAME' => client_name,
           'KNAPSACK-PRO-CLIENT-VERSION' => KnapsackPro::VERSION,
+          'KNAPSACK-PRO-TEST-SUITE-TOKEN' => test_suite_token,
         }
       end
 
@@ -95,19 +92,15 @@ module KnapsackPro
         !seed.nil?
       end
 
-      def post
+      def make_request(&block)
         retries ||= 0
-        uri = URI.parse(endpoint_url)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = (uri.scheme == 'https')
-        http.open_timeout = TIMEOUT
-        http.read_timeout = TIMEOUT
 
-        @http_response = http.post(uri.path, request_body, json_headers)
+        @http_response = block.call
         @response_body = parse_response_body(http_response.body)
 
         request_uuid = http_response.header['X-Request-Id'] || 'N/A'
 
+        logger.debug("#{action.http_method.to_s.upcase} #{endpoint_url}")
         logger.debug("API request UUID: #{request_uuid}")
         logger.debug("Test suite split seed: #{seed}") if has_seed?
         logger.debug('API response:')
@@ -125,7 +118,7 @@ module KnapsackPro
       rescue ServerError, Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::EPIPE, EOFError, SocketError, Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError => e
         logger.warn(e.inspect)
         retries += 1
-        if retries < MAX_RETRY
+        if retries < MAX_RETRY.call
           wait = retries * REQUEST_RETRY_TIMEBOX
           logger.warn("Wait #{wait}s and retry request to Knapsack Pro API.")
           Kernel.sleep(wait)
@@ -134,6 +127,31 @@ module KnapsackPro
           response_body
         end
       end
+
+      def build_http(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == 'https')
+        http.open_timeout = TIMEOUT
+        http.read_timeout = TIMEOUT
+        http
+      end
+
+      def post
+        uri = URI.parse(endpoint_url)
+        http = build_http(uri)
+        make_request do
+          http.post(uri.path, request_body, json_headers)
+        end
+      end
+
+      def get
+        uri = URI.parse(endpoint_url)
+        uri.query = URI.encode_www_form(request_hash)
+        http = build_http(uri)
+        make_request do
+          http.get(uri, json_headers)
+        end
+      end
     end
   end
 end

data/lib/knapsack_pro/config/env.rb

@@ -55,6 +55,10 @@ module KnapsackPro
           ENV['KNAPSACK_PRO_TEST_FILE_PATTERN']
         end
 
+        def slow_test_file_pattern
+          ENV['KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN']
+        end
+
         def test_file_exclude_pattern
           ENV['KNAPSACK_PRO_TEST_FILE_EXCLUDE_PATTERN']
         end

data/lib/knapsack_pro/formatters/rspec_json_formatter.rb

@@ -4,7 +4,7 @@ RSpec::Support.require_rspec_core('formatters/json_formatter')
 module KnapsackPro
   module Formatters
     class RSpecJsonFormatter < RSpec::Core::Formatters::JsonFormatter
-      RSpec::Core::Formatters.register self, :message, :dump_summary, :dump_profile, :stop, :seed, :close
+      RSpec::Core::Formatters.register self
 
       private
 

data/lib/knapsack_pro/queue_allocator.rb

@@ -1,7 +1,8 @@
 module KnapsackPro
   class QueueAllocator
     def initialize(args)
-      @test_files = args.fetch(:test_files)
+      @fast_and_slow_test_files_to_run = args.fetch(:fast_and_slow_test_files_to_run)
+      @fallback_mode_test_files = args.fetch(:fallback_mode_test_files)
       @ci_node_total = args.fetch(:ci_node_total)
       @ci_node_index = args.fetch(:ci_node_index)
       @ci_node_build_id = args.fetch(:ci_node_build_id)
@@ -36,14 +37,15 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_files,
+    attr_reader :fast_and_slow_test_files_to_run,
+      :fallback_mode_test_files,
       :ci_node_total,
       :ci_node_index,
       :ci_node_build_id,
       :repository_adapter
 
     def encrypted_test_files
-      KnapsackPro::Crypto::Encryptor.call(test_files)
+      KnapsackPro::Crypto::Encryptor.call(fast_and_slow_test_files_to_run)
     end
 
     def encrypted_branch
@@ -63,12 +65,12 @@ module KnapsackPro
     end
 
     def prepare_test_files(response)
-      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(test_files, response['test_files'])
+      decrypted_test_files = KnapsackPro::Crypto::Decryptor.call(fast_and_slow_test_files_to_run, response['test_files'])
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
 
     def fallback_test_files(executed_test_files)
-      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(test_files, ci_node_total)
+      test_flat_distributor = KnapsackPro::TestFlatDistributor.new(fallback_mode_test_files, ci_node_total)
       test_files_for_node_index = test_flat_distributor.test_files_for_node(ci_node_index)
       KnapsackPro::TestFilePresenter.paths(test_files_for_node_index) - executed_test_files
     end

data/lib/knapsack_pro/queue_allocator_builder.rb

@@ -2,7 +2,8 @@ module KnapsackPro
   class QueueAllocatorBuilder < BaseAllocatorBuilder
     def allocator
       KnapsackPro::QueueAllocator.new(
-        test_files: test_files,
+        fast_and_slow_test_files_to_run: fast_and_slow_test_files_to_run,
+        fallback_mode_test_files: fallback_mode_test_files,
         ci_node_total: env.ci_node_total,
         ci_node_index: env.ci_node_index,
         ci_node_build_id: env.ci_node_build_id,

data/lib/knapsack_pro/report.rb

@@ -37,7 +37,7 @@ module KnapsackPro
       if test_files.empty?
         KnapsackPro.logger.warn("No test files were executed on this CI node.")
         KnapsackPro.logger.debug("When you use knapsack_pro queue mode then probably reason might be that CI node was started after the test files from the queue were already executed by other CI nodes. That is why this CI node has no test files to execute.")
-        KnapsackPro.logger.debug("Another reason might be when your CI node failed in a way that prevented knapsack_pro to save time execution data to Knapsack Pro API and you have just tried to retry failed CI node but instead you got no test files to execute. In that case knapsack_pro don't know what testes should be executed here.")
+        KnapsackPro.logger.debug("Another reason might be when your CI node failed in a way that prevented knapsack_pro to save time execution data to Knapsack Pro API and you have just tried to retry failed CI node but instead you got no test files to execute. In that case knapsack_pro don't know what tests should be executed here.")
       end
 
       measured_test_files = test_files

data/lib/knapsack_pro/runners/queue/cucumber_runner.rb

@@ -91,7 +91,16 @@ module KnapsackPro
           # which is defined in lib/knapsack_pro/adapters/cucumber_adapter.rb
           ENV['KNAPSACK_PRO_BEFORE_QUEUE_HOOK_CALLED'] = 'true'
 
-          $?.exitstatus
+          process_status = $?
+
+          unless process_status.exited?
+            raise "Cucumber process execution failed. It's likely that your CI server has exceeded"\
+                    " its available memory. Please try changing CI config or retrying the CI build.\n"\
+                    "Failed command: #{cmd}\n"\
+                    "Process status: #{process_status.inspect}"
+          end
+
+          process_status.exitstatus
         end
       end
     end

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -134,6 +134,13 @@ module KnapsackPro
           RSpec.world.example_groups.clear
           RSpec.configuration.start_time = ::RSpec::Core::Time.now
 
+          if KnapsackPro::Config::Env.rspec_split_by_test_examples?
+            # Reset example group counts to ensure scoped example ids in metadata
+            # have correct index (not increased by each subsequent run).
+            # Solves this problem: https://github.com/rspec/rspec-core/issues/2721
+            RSpec.world.instance_variable_set(:@example_group_counts_by_spec_file, Hash.new(0))
+          end
+
           # skip reset filters for old RSpec versions
           if RSpec.configuration.respond_to?(:reset_filters)
             RSpec.configuration.reset_filters

data/lib/knapsack_pro/runners/rspec_runner.rb

@@ -16,8 +16,11 @@ module KnapsackPro
           end
 
           RSpec::Core::RakeTask.new(task_name) do |t|
-            t.rspec_opts = "#{args} --default-path #{runner.test_dir}"
-            t.pattern = runner.test_file_paths
+            # we cannot pass runner.test_file_paths array to t.pattern
+            # because pattern does not accept test example path like spec/a_spec.rb[1:2]
+            # instead we pass test files and test example paths to t.rspec_opts
+            t.pattern = []
+            t.rspec_opts = "#{args} --default-path #{runner.test_dir} #{runner.stringify_test_file_paths}"
           end
           Rake::Task[task_name].invoke
         end

data/lib/knapsack_pro/slow_test_file_determiner.rb

@@ -0,0 +1,33 @@
+module KnapsackPro
+  class SlowTestFileDeterminer
+    TIME_THRESHOLD_PER_CI_NODE = 0.7 # 70%
+    REPORT_DIR = 'tmp/knapsack_pro/slow_test_file_determiner'
+
+    # test_files: { 'path' => 'a_spec.rb', 'time_execution' => 0.0 }
+    # time_execution: of build distribution (total time of CI build run)
+    def self.call(test_files, time_execution)
+      time_threshold = (time_execution / KnapsackPro::Config::Env.ci_node_total) * TIME_THRESHOLD_PER_CI_NODE
+
+      test_files.select do |test_file|
+        test_file.fetch('time_execution') >= time_threshold
+      end
+    end
+
+    def self.save_to_json_report(test_files)
+      FileUtils.mkdir_p(REPORT_DIR)
+      File.write(report_path, test_files.to_json)
+    end
+
+    def self.read_from_json_report
+      raise 'Report with slow test files was not generated yet. If you have enabled split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases and you see this error it means that your tests accidentally cleaned up tmp/knapsack_pro directory. Please do not remove this directory during tests runtime!' unless File.exists?(report_path)
+      slow_test_files_json_report = File.read(report_path)
+      JSON.parse(slow_test_files_json_report)
+    end
+
+    private
+
+    def self.report_path
+      "#{REPORT_DIR}/slow_test_files_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+    end
+  end
+end

data/lib/knapsack_pro/slow_test_file_finder.rb

@@ -0,0 +1,27 @@
+module KnapsackPro
+  class SlowTestFileFinder
+    # Get recorded test files from API.
+    # Find slow tests among them that are still present on the disk.
+    # Save slow test files in json file on the disk.
+    # Returns slow test files.
+    def self.call(adapter_class)
+      if KnapsackPro::Config::Env.test_files_encrypted?
+        raise 'Split by test cases is not possible when you have enabled test file names encryption ( https://github.com/KnapsackPro/knapsack_pro-ruby#test-file-names-encryption ). You need to disable encryption with KNAPSACK_PRO_TEST_FILES_ENCRYPTED=false in order to use split by test cases https://github.com/KnapsackPro/knapsack_pro-ruby#split-test-files-by-test-cases'
+      end
+
+      # get list of recorded test files for last CI Build
+      build_distribution_entity = KnapsackPro::BuildDistributionFetcher.call
+      test_files_from_api = build_distribution_entity.test_files
+
+      merged_test_files_from_api = KnapsackPro::TestCaseMergers::BaseMerger.call(adapter_class, test_files_from_api)
+
+      test_files_existing_on_disk = KnapsackPro::TestFileFinder.select_test_files_that_can_be_run(adapter_class, merged_test_files_from_api)
+
+      slow_test_files = KnapsackPro::SlowTestFileDeterminer.call(test_files_existing_on_disk, build_distribution_entity.time_execution)
+
+      KnapsackPro::SlowTestFileDeterminer.save_to_json_report(slow_test_files)
+
+      slow_test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_detectors/rspec_test_example_detector.rb

@@ -2,7 +2,6 @@ module KnapsackPro
   module TestCaseDetectors
     class RSpecTestExampleDetector
       REPORT_DIR = 'tmp/knapsack_pro/test_case_detectors/rspec'
-      REPORT_PATH = "#{REPORT_DIR}/rspec_dry_run_json_report.json"
 
       def generate_json_report
         require 'rspec/core'
@@ -18,13 +17,19 @@ module KnapsackPro
         ensure_report_dir_exists
         remove_old_json_report
 
-        test_file_paths = KnapsackPro::TestFileFinder.call(test_file_pattern)
+        test_file_entities = slow_test_files
+
+        if test_file_entities.empty?
+          no_examples_json = { examples: [] }.to_json
+          File.write(report_path, no_examples_json)
+          return
+        end
 
         cli_args = cli_format + [
           '--dry-run',
-          '--out', REPORT_PATH,
+          '--out', report_path,
           '--default-path', test_dir,
-        ] + test_file_paths.map { |t| t.fetch('path') }
+        ] + KnapsackPro::TestFilePresenter.paths(test_file_entities)
         options = RSpec::Core::ConfigurationOptions.new(cli_args)
         exit_code = RSpec::Core::Runner.new(options).run($stderr, $stdout)
         if exit_code != 0
@@ -33,9 +38,9 @@ module KnapsackPro
       end
 
       def test_file_example_paths
-        raise "No report found at #{REPORT_PATH}" unless File.exists?(REPORT_PATH)
+        raise "No report found at #{report_path}" unless File.exists?(report_path)
 
-        json_report = File.read(REPORT_PATH)
+        json_report = File.read(report_path)
         hash_report = JSON.parse(json_report)
         hash_report
           .fetch('examples')
@@ -43,8 +48,22 @@ module KnapsackPro
           .map { |path_with_example_id| test_file_hash_for(path_with_example_id) }
       end
 
+      def slow_test_files
+        if KnapsackPro::Config::Env.slow_test_file_pattern
+          KnapsackPro::TestFileFinder.slow_test_files_by_pattern(adapter_class)
+        else
+          # read slow test files from JSON file on disk that was generated
+          # by lib/knapsack_pro/base_allocator_builder.rb
+          KnapsackPro::SlowTestFileDeterminer.read_from_json_report
+        end
+      end
+
       private
 
+      def report_path
+        "#{REPORT_DIR}/rspec_dry_run_json_report_node_#{KnapsackPro::Config::Env.ci_node_index}.json"
+      end
+
       def adapter_class
         KnapsackPro::Adapters::RSpecAdapter
       end
@@ -62,7 +81,7 @@ module KnapsackPro
       end
 
       def remove_old_json_report
-        File.delete(REPORT_PATH) if File.exists?(REPORT_PATH)
+        File.delete(report_path) if File.exists?(report_path)
       end
 
       def test_file_hash_for(test_file_path)

data/lib/knapsack_pro/test_case_mergers/base_merger.rb

@@ -0,0 +1,29 @@
+module KnapsackPro
+  module TestCaseMergers
+    class BaseMerger
+      # values must be string to avoid circular dependency problem during loading files
+      ADAPTER_TO_MERGER_MAP = {
+        KnapsackPro::Adapters::RSpecAdapter => 'KnapsackPro::TestCaseMergers::RSpecMerger',
+      }
+
+      def self.call(adapter_class, test_files)
+        merger_class =
+          ADAPTER_TO_MERGER_MAP[adapter_class] ||
+          raise("Test case merger does not exist for adapter_class: #{adapter_class}")
+        Kernel.const_get(merger_class).new(test_files).call
+      end
+
+      def initialize(test_files)
+        @test_files = test_files
+      end
+
+      def call
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_reader :test_files
+    end
+  end
+end

data/lib/knapsack_pro/test_case_mergers/rspec_merger.rb

@@ -0,0 +1,34 @@
+module KnapsackPro
+  module TestCaseMergers
+    class RSpecMerger < BaseMerger
+      def call
+        merged_test_files_hash = {}
+        test_files.each do |test_file|
+          test_file_path = extract_test_file_path(test_file.fetch('path'))
+
+          # must be float (default type for time execution from API)
+          merged_test_files_hash[test_file_path] ||= 0.0
+          merged_test_files_hash[test_file_path] += test_file.fetch('time_execution')
+        end
+
+        merged_test_files = []
+        merged_test_files_hash.each do |path, time_execution|
+          merged_test_files << {
+            'path' => path,
+            'time_execution' => time_execution
+          }
+        end
+        merged_test_files
+      end
+
+      private
+
+      # path - can be:
+      # test file path: spec/a_spec.rb
+      # or test example path: spec/a_spec.rb[1:1]
+      def extract_test_file_path(path)
+        path.gsub(/\.rb\[.+\]$/, '.rb')
+      end
+    end
+  end
+end

data/lib/knapsack_pro/test_file_finder.rb

@@ -1,11 +1,49 @@
 module KnapsackPro
   class TestFileFinder
-    def self.call(test_file_pattern)
-      new(test_file_pattern).call
+    def self.call(test_file_pattern, test_file_list_enabled: true)
+      new(test_file_pattern, test_file_list_enabled).call
     end
 
-    def initialize(test_file_pattern)
+    # finds slow test files on disk based on ENV patterns
+    # returns example: [{ 'path' => 'a_spec.rb' }]
+    def self.slow_test_files_by_pattern(adapter_class)
+      raise 'KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN is not defined' unless KnapsackPro::Config::Env.slow_test_file_pattern
+
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      slow_test_file_entities = call(KnapsackPro::Config::Env.slow_test_file_pattern, test_file_list_enabled: false)
+
+      # slow test files (KNAPSACK_PRO_SLOW_TEST_FILE_PATTERN)
+      # should be subset of test file pattern (KNAPSACK_PRO_TEST_FILE_PATTERN)
+      slow_test_file_entities & test_file_entities
+    end
+
+    # Args:
+    #   test_file_entities_to_run - it can be list of slow test files that you want to run
+    # Return:
+    #   subset of test_file_entities_to_run that are present on disk and it is subset of tests matching pattern KNAPSACK_PRO_TEST_FILE_PATTERN
+    #   Thanks to that we can select only slow test files that are within list of test file pattern we want to run tests for
+    def self.select_test_files_that_can_be_run(adapter_class, test_file_entities_to_run)
+      test_file_pattern = KnapsackPro::TestFilePattern.call(adapter_class)
+      test_file_entities = call(test_file_pattern)
+
+      test_file_paths_existing_on_disk = KnapsackPro::TestFilePresenter.paths(test_file_entities)
+
+      selected_test_files = []
+
+      test_file_entities_to_run.each do |test_file_entity|
+        if test_file_paths_existing_on_disk.include?(test_file_entity.fetch('path'))
+          selected_test_files << test_file_entity
+        end
+      end
+
+      selected_test_files
+    end
+
+    def initialize(test_file_pattern, test_file_list_enabled)
       @test_file_pattern = test_file_pattern
+      @test_file_list_enabled = test_file_list_enabled
     end
 
     def call
@@ -18,10 +56,10 @@ module KnapsackPro
 
     private
 
-    attr_reader :test_file_pattern
+    attr_reader :test_file_pattern, :test_file_list_enabled
 
     def test_files
-      if KnapsackPro::Config::Env.test_file_list
+      if test_file_list_enabled && KnapsackPro::Config::Env.test_file_list
         return KnapsackPro::Config::Env.test_file_list.split(',').map(&:strip)
       end