knapsack_pro 9.2.2 → 10.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53d7f0f50c7114a8e5cefd993a8ef8cb5fa05a6d36dd78cf8f9784884a6ac5d9
-  data.tar.gz: 56366ca74436f87208e2a297941d00ec403589e0e6d0ed3759862d059b8b16bb
+  metadata.gz: 142cab6d40b7b31270887dd59d36daa249813769e01ee5a83aba4b78dc4cea1d
+  data.tar.gz: ee60df331664ef4bac5e3aa7906cdf41f4a5c9c69a5ebada6c542a6dc2780313
 SHA512:
-  metadata.gz: 315e0df100e135cdd2646b546ce331291514c24d40f0ff88ea0a69f0d60842623cc35cb8f390a07b725c6b89ecbbebe61cf587a067728c2d2c59c99f531df4f0
-  data.tar.gz: 8d51be76e65fce53d2cdae3c45359ad5ce1dcdfab0a48c6ccacef0fe4d85636879c2586d1f937853f7d3d7b4652846e182af4fa2a548d9e2405399a538695599
+  metadata.gz: 5591eb67604230bd3f33cbc82f0662b8dd0a289834166760262ca44df22ae83651599f9294faa3a31e31c25cdc743a1b3870829bbac429c15364bbfdb28349dc
+  data.tar.gz: 3a248092779d5459d8776b94963b745f798cd55d57137764aa54b9a58ba9688ca75f349c35e0c165c6e2c8452803caa13b6cb0f5f73527e76d58354a1f364d2d

data/exe/knapsack_pro

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "knapsack_pro"
+require "knapsack_pro/commands"
+KnapsackPro::Commands.start(ARGV)

data/lib/knapsack_pro/adapters/rspec_adapter.rb

@@ -81,11 +81,15 @@ module KnapsackPro
         !id.nil?
       end
 
+      def self.concat_test_files(test_files, id_paths)
+        paths = concat_paths(test_files, id_paths)
+        KnapsackPro::TestFilePresenter.test_files(paths)
+      end
+
       def self.concat_paths(test_files, id_paths)
         paths = KnapsackPro::TestFilePresenter.paths(test_files)
         file_paths = id_paths.map { |id_path| parse_file_path(id_path) }
-        acc = paths + id_paths - file_paths
-        KnapsackPro::TestFilePresenter.test_files(acc)
+        paths + id_paths - file_paths
       end
 
       def self.rails_helper_exists?(test_dir)
@@ -138,7 +142,7 @@ module KnapsackPro
         ::RSpec.configure do |config|
           config.append_before(:suite) do
             time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
-            time_tracker.scheduled_paths = KnapsackPro::Adapters::RSpecAdapter.scheduled_paths
+            time_tracker.schedule(KnapsackPro::Adapters::RSpecAdapter.scheduled_paths)
           end
         end
       end

data/lib/knapsack_pro/build_distribution_fetcher.rb

@@ -59,7 +59,7 @@ module KnapsackPro
 
     def additional_params
       {
-        node_build_id: KnapsackPro::Config::Env.ci_node_build_id,
+        node_build_id: KnapsackPro::Config::Env.test_queue_id,
         none_if_queue_initialized: true
       }
     end

data/lib/knapsack_pro/client/api/action.rb

@@ -3,15 +3,7 @@
 module KnapsackPro
   module Client
     module API
-      class Action
-        attr_reader :endpoint_path, :http_method, :request_hash
-
-        def initialize(args)
-          @endpoint_path = args.fetch(:endpoint_path)
-          @http_method = args.fetch(:http_method)
-          @request_hash = args.fetch(:request_hash)
-        end
-      end
+      Action = Struct.new(:endpoint_path, :http_method, :request_hash, keyword_init: true)
     end
   end
 end

data/lib/knapsack_pro/client/api/v1/queues.rb

@@ -36,59 +36,6 @@ module KnapsackPro
                 request_hash: request_hash
               )
             end
-
-            def initialize(paths)
-              git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
-              repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
-
-              request_hash = {
-                attempt_connect_to_queue: false,
-                branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
-                build_author: git_adapter.build_author,
-                can_initialize_queue: true,
-                commit_authors: git_adapter.commit_authors,
-                commit_hash: repository_adapter.commit_hash,
-                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split,
-                node_build_id: KnapsackPro::Config::Env.ci_node_build_id,
-                node_index: KnapsackPro::Config::Env.ci_node_index,
-                node_total: KnapsackPro::Config::Env.ci_node_total,
-                skip_pull: true,
-                test_files: KnapsackPro::Crypto::Encryptor.call(paths),
-                user_seat: KnapsackPro::Config::Env.masked_user_seat,
-              }
-
-              action_class.new(
-                endpoint_path: '/v1/queues/queue',
-                http_method: :post,
-                request_hash: request_hash
-              )
-            end
-
-            def connect
-              git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
-              repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
-
-              request_hash = {
-                attempt_connect_to_queue: true,
-                branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
-                build_author: git_adapter.build_author,
-                can_initialize_queue: true,
-                commit_authors: git_adapter.commit_authors,
-                commit_hash: repository_adapter.commit_hash,
-                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split,
-                node_build_id: KnapsackPro::Config::Env.ci_node_build_id,
-                node_index: KnapsackPro::Config::Env.ci_node_index,
-                node_total: KnapsackPro::Config::Env.ci_node_total,
-                skip_pull: true,
-                user_seat: KnapsackPro::Config::Env.masked_user_seat,
-              }
-
-              action_class.new(
-                endpoint_path: '/v1/queues/queue',
-                http_method: :post,
-                request_hash: request_hash
-              )
-            end
           end
         end
       end

data/lib/knapsack_pro/client/api/v2/queues.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  module Client
+    module API
+      module V2
+        class Queues < KnapsackPro::Client::API::V1::Base
+          CODE_ATTEMPT_CONNECT_TO_QUEUE_FAILED = 'ATTEMPT_CONNECT_TO_QUEUE_FAILED'
+
+          class << self
+            def queue(args)
+              request_hash = {
+                attempt_connect_to_queue: args.fetch(:attempt_connect_to_queue),
+                batch_index: args.fetch(:batch_index),
+                branch: args.fetch(:branch),
+                can_initialize_queue: args.fetch(:can_initialize_queue),
+                commit_hash: args.fetch(:commit_hash),
+                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split_?,
+                node_index: args.fetch(:node_index),
+                node_total: args.fetch(:node_total),
+                node_uuid: KnapsackPro::Config::Env.node_uuid,
+                test_queue_id: KnapsackPro::Config::Env.test_queue_id,
+                user_seat: KnapsackPro::Config::Env.masked_user_seat
+              }
+
+              if request_hash[:can_initialize_queue] && !request_hash[:attempt_connect_to_queue]
+                git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
+
+                request_hash.merge!(
+                  build_author: git_adapter.build_author,
+                  commit_authors: git_adapter.commit_authors,
+                  paths: args.fetch(:paths)
+                )
+              end
+
+              if !request_hash[:can_initialize_queue] && !request_hash[:attempt_connect_to_queue]
+                request_hash.merge!(
+                  failed_paths: args.fetch(:failed_paths),
+                  batch_id: args.fetch(:batch_id)
+                )
+              end
+
+              action_class.new(
+                endpoint_path: '/v2/queues/queue',
+                http_method: :post,
+                request_hash: request_hash
+              )
+            end
+
+            def connect
+              git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
+              repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
+
+              request_hash = {
+                attempt_connect_to_queue: true,
+                batch_index: 0,
+                branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
+                build_author: git_adapter.build_author,
+                can_initialize_queue: true,
+                commit_authors: git_adapter.commit_authors,
+                commit_hash: repository_adapter.commit_hash,
+                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split_?,
+                node_index: KnapsackPro::Config::Env.ci_node_index,
+                node_total: KnapsackPro::Config::Env.ci_node_total,
+                node_uuid: SecureRandom.uuid,
+                skip_pull: true,
+                test_queue_id: KnapsackPro::Config::Env.test_queue_id,
+                user_seat: KnapsackPro::Config::Env.masked_user_seat,
+              }
+
+              action_class.new(
+                endpoint_path: '/v2/queues/queue',
+                http_method: :post,
+                request_hash: request_hash
+              )
+            end
+
+            def initialize(paths)
+              git_adapter = KnapsackPro::RepositoryAdapters::GitAdapter.new
+              repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
+
+              request_hash = {
+                attempt_connect_to_queue: false,
+                batch_index: 0,
+                branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
+                build_author: git_adapter.build_author,
+                can_initialize_queue: true,
+                commit_authors: git_adapter.commit_authors,
+                commit_hash: repository_adapter.commit_hash,
+                fixed_queue_split: KnapsackPro::Config::Env.fixed_queue_split_?,
+                node_index: KnapsackPro::Config::Env.ci_node_index,
+                node_total: KnapsackPro::Config::Env.ci_node_total,
+                node_uuid: SecureRandom.uuid,
+                skip_pull: true,
+                paths: KnapsackPro::Crypto::Encryptor.paths(paths),
+                test_queue_id: KnapsackPro::Config::Env.test_queue_id,
+                user_seat: KnapsackPro::Config::Env.masked_user_seat,
+              }
+
+              action_class.new(
+                endpoint_path: '/v2/queues/queue',
+                http_method: :post,
+                request_hash: request_hash
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/knapsack_pro/client/connection.rb

@@ -86,9 +86,9 @@ module KnapsackPro
       end
 
       def make_request(&block)
-        retries ||= 0
+        attempt_count ||= 1
 
-        @http_response = block.call
+        @http_response = block.call(attempt_count)
         @response_body = parse_response_body(http_response.body)
 
         request_uuid = http_response.header['X-Request-Id'] || 'N/A'
@@ -108,11 +108,11 @@ module KnapsackPro
         response_body
       rescue ServerError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EPIPE, EOFError,
              SocketError, Net::OpenTimeout, Net::ReadTimeout, OpenSSL::SSL::SSLError => e
-        retries += 1
-        log_diagnostics(e, retries)
-        @http.set_debug_output(@http_debug_output) if retries == max_request_retries - 1
-        if retries < max_request_retries
-          backoff(retries)
+        attempt_count += 1
+        log_diagnostics(e, attempt_count)
+        @http.set_debug_output(@http_debug_output) if attempt_count == max_request_retries
+        if attempt_count <= max_request_retries
+          backoff(attempt_count)
           rotate_ip
           retry
         else
@@ -120,7 +120,7 @@ module KnapsackPro
         end
       end
 
-      def log_diagnostics(error, retries)
+      def log_diagnostics(error, attempt_count)
         message = [
           action.http_method.to_s.upcase,
           endpoint_uri,
@@ -129,7 +129,7 @@ module KnapsackPro
         logger.warn(message)
         logger.warn('Request failed due to:')
         logger.warn(error.inspect)
-        return if retries < max_request_retries
+        return if attempt_count < max_request_retries + 1
 
         error.backtrace.each { |line| logger.warn(line) }
         logger.warn('Net::HTTP debug output:')
@@ -156,8 +156,8 @@ module KnapsackPro
         end
       end
 
-      def backoff(retries)
-        wait = retries * REQUEST_RETRY_TIMEBOX
+      def backoff(attempt_count)
+        wait = (attempt_count - 1) * REQUEST_RETRY_TIMEBOX
         print_every = 2 # seconds
         (wait / print_every).ceil.times do |i|
           if i.zero?
@@ -204,8 +204,8 @@ module KnapsackPro
 
       def post
         build_http(endpoint_uri)
-        make_request do
-          @http.post(endpoint_uri.path, action.request_hash.to_json, json_headers)
+        make_request do |attempt_count|
+          @http.post(endpoint_uri.path, action.request_hash.merge(attempt_count: attempt_count).to_json, json_headers)
         end
       end
 
@@ -213,7 +213,7 @@ module KnapsackPro
         uri = endpoint_uri
         uri.query = URI.encode_www_form(action.request_hash)
         build_http(uri)
-        make_request do
+        make_request do |_attempt_count|
           @http.get(uri, json_headers)
         end
       end
@@ -231,7 +231,7 @@ module KnapsackPro
         return 6 if KnapsackPro::Config::Env.regular_mode?
 
         # default number of attempts
-        3
+        4
       end
     end
   end

data/lib/knapsack_pro/commands/retry_failed_tests.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+
+module KnapsackPro
+  class RetryFailedTests
+    def initialize(branch)
+      @branch = branch || `git branch --show-current`.chomp
+    end
+
+    def call(runner_args)
+      $stderr.puts "Branch: #{branch}"
+
+      failed_paths = fetch_failed_paths
+      return ($stderr.puts "Nothing to run") if failed_paths.size.zero?
+
+      $stderr.puts "Retrying #{failed_paths.size} tests..."
+      exec Gem.bin_path("rspec-core", "rspec"), *(runner_args + failed_paths)
+    end
+
+    private
+
+    attr_accessor :branch
+
+    def fetch_failed_paths
+      headers = {
+        'Accept' => 'application/json',
+        'KNAPSACK-PRO-TEST-SUITE-TOKEN' => ENV.fetch("KNAPSACK_PRO_TEST_SUITE_TOKEN")
+      }
+
+      uri = URI.parse("#{KnapsackPro::Config::Env.endpoint}/v2/test_paths")
+      uri.query = URI.encode_www_form(branch: KnapsackPro::Crypto::BranchEncryptor.call(branch))
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == 'https')
+      http.open_timeout = 5
+      http.read_timeout = 5
+
+      response = http.get(uri.request_uri, headers)
+      abort response.inspect if (300..).cover?(response.code.to_i)
+
+      parsed = JSON.parse(response.body)
+      abort parsed.inspect if parsed['errors']
+
+      parsed.fetch('failed_paths')
+    end
+  end
+end

data/lib/knapsack_pro/commands.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module KnapsackPro
+  class Commands < Thor
+    map "queue:rspec" => :queue_rspec
+    map "queue:cucumber" => :queue_cucumber
+    map "queue:minitest" => :queue_minitest
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "rspec ['arguments']", "Parallelize RSpec with Knapsack Pro in Regular Mode"
+    def rspec(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::RSpecRunner.run(arguments)
+    end
+
+    desc "queue:rspec ['arguments']", "Parallelize RSpec with Knapsack Pro in Queue Mode"
+    def queue_rspec(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::Queue::RSpecRunner.run(arguments)
+    end
+
+    desc "cucumber ['arguments']", "Parallelize Cucumber with Knapsack Pro in Regular Mode"
+    def cucumber(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::CucumberRunner.run(arguments)
+    end
+
+    desc "queue:cucumber ['arguments']", "Parallelize Cucumber with Knapsack Pro in Queue Mode"
+    def queue_cucumber(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::Queue::CucumberRunner.run(arguments)
+    end
+
+    desc "minitest ['arguments']", "Parallelize Minitest with Knapsack Pro in Regular Mode"
+    def minitest(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::MinitestRunner.run(arguments)
+    end
+
+    desc "queue:minitest ['arguments']", "Parallelize Minitest with Knapsack Pro in Queue Mode"
+    def queue_minitest(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::Queue::MinitestRunner.run(arguments)
+    end
+
+    desc "test_unit ['arguments']", "Parallelize TestUnit with Knapsack Pro in Regular Mode"
+    def test_unit(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::TestUnitRunner.run(arguments)
+    end
+
+    desc "spinach ['arguments']", "Parallelize Spinach with Knapsack Pro in Regular Mode"
+    def spinach(arguments = "")
+      require "knapsack_pro"
+      KnapsackPro::Runners::SpinachRunner.run(arguments)
+    end
+
+    desc "retry [-b] [-- test_runner_args]", "Retry RSpec the tests that failed on the previous Knapsack Pro run on BRANCH."
+    long_desc <<~DESC
+      \x5knapsack_pro retry
+      \x5knapsack_pro retry --branch feature -- --format progress
+    DESC
+    option :branch, type: :string, aliases: :b, desc: "Default: current branch."
+    def retry(*runner_args)
+      require_relative "./commands/retry_failed_tests"
+      KnapsackPro::RetryFailedTests.new(options[:branch]).call(runner_args)
+    end
+  end
+end

data/lib/knapsack_pro/config/ci/base.rb

@@ -49,6 +49,9 @@ module KnapsackPro
 
           nil
         end
+
+        def test_queue_id
+        end
       end
     end
   end

data/lib/knapsack_pro/config/ci/buildkite.rb

@@ -47,6 +47,10 @@ module KnapsackPro
         def ci_provider
           "Buildkite"
         end
+
+        def test_queue_id
+          node_build_id
+        end
       end
     end
   end

data/lib/knapsack_pro/config/ci/circle.rb

@@ -43,6 +43,12 @@ module KnapsackPro
         def ci_provider
           "CircleCI"
         end
+
+        def test_queue_id
+          # CIRCLE_PIPELINE_NUMBER does not exist in Circle, set it with:
+          # `CIRCLE_PIPELINE_NUMBER: << pipeline.number >>`
+          ENV['CIRCLE_PIPELINE_NUMBER']
+        end
       end
     end
   end

data/lib/knapsack_pro/config/ci/github_actions.rb

@@ -57,6 +57,10 @@ module KnapsackPro
         def ci_provider
           "GitHub Actions"
         end
+
+        def test_queue_id
+          node_build_id
+        end
       end
     end
   end

data/lib/knapsack_pro/config/ci/gitlab_ci.rb

@@ -51,6 +51,10 @@ module KnapsackPro
         def ci_provider
           "Gitlab CI"
         end
+
+        def test_queue_id
+          ENV['CI_PIPELINE_ID']
+        end
       end
     end
   end

data/lib/knapsack_pro/config/env.rb

@@ -164,6 +164,12 @@ module KnapsackPro
           end
         end
 
+        def fixed_queue_split_?
+          return @fixed_queue_split_ if defined?(@fixed_queue_split_)
+
+          @fixed_queue_split_ = env_for('KNAPSACK_PRO_FIXED_QUEUE_SPLIT', :fixed_queue_split).to_s == "true"
+        end
+
         def fixed_queue_split?
           fixed_queue_split.to_s == 'true'
         end
@@ -260,6 +266,30 @@ module KnapsackPro
           ENV.fetch('KNAPSACK_PRO_FALLBACK_MODE_ERROR_EXIT_CODE', 1).to_i
         end
 
+        def test_queue_id
+          knapsack_env_name = 'KNAPSACK_PRO_TEST_QUEUE_ID'
+          knapsack_env_value = ENV[knapsack_env_name]
+
+          ci_env_value = detected_ci.new.test_queue_id
+
+          if !knapsack_env_value.nil? && !ci_env_value.nil? && knapsack_env_value != ci_env_value.to_s
+            warn("You have set the environment variable #{knapsack_env_name} to #{knapsack_env_value} which could be automatically determined from the CI environment as #{ci_env_value}.")
+          end
+
+          id = knapsack_env_value != nil ? knapsack_env_value : ci_env_value
+          return id unless id.nil?
+
+          triplet = [ci_node_total, branch, commit_hash]
+          return triplet.join('-') if triplet.all?
+
+          raise("Missing test_queue_id. See: #{KnapsackPro::Urls::KNAPSACK_PRO_TEST_QUEUE_ID}")
+        end
+
+        def node_uuid
+          env_name = 'KNAPSACK_PRO_NODE_UUID'
+          ENV[env_name] || raise("Missing environment variable #{env_name}. Please report this as a bug: #{KnapsackPro::Urls::SUPPORT}")
+        end
+
         private
 
         def required_env(env_name)

data/lib/knapsack_pro/crypto/encryptor.rb

@@ -11,6 +11,14 @@ module KnapsackPro
         end
       end
 
+      def self.paths(paths)
+        if KnapsackPro::Config::Env.test_files_encrypted?
+          paths.map { |path| Digestor.salt_hexdigest(path) }
+        else
+          paths
+        end
+      end
+
       def initialize(test_files)
         @test_files = test_files
       end

data/lib/knapsack_pro/formatters/time_tracker.rb

@@ -23,17 +23,37 @@ module KnapsackPro
         @group = {}
         @paths = {}
         @suite_started = now
-        @scheduled_paths = []
+        @batched_scheduled_paths = []
         @split_by_test_example_file_paths = Set.new
+        @current_batch_examples = []
       end
 
-      def scheduled_paths=(scheduled_paths)
-        @scheduled_paths = scheduled_paths
-        @scheduled_paths.each do |path|
-          if KnapsackPro::Adapters::RSpecAdapter.id_path?(path)
-            file_path = KnapsackPro::Adapters::RSpecAdapter.parse_file_path(path)
-            @split_by_test_example_file_paths << file_path
+      def current_batch_failed_paths
+        examples_by_file_path = @current_batch_examples
+          .group_by { |example| file_path_for(example) }
+
+        paths =
+          examples_by_file_path.flat_map do |file_path, examples|
+            failed_id_paths = examples.filter { |example| example.execution_result.status.to_s == "failed" }.map(&:id)
+
+            # Other nodes may have run some examples from this file, it's not safe to compact.
+            next failed_id_paths if rspec_split_by_test_example?(file_path)
+            next file_path if failed_id_paths.size == examples.size
+            failed_id_paths
           end
+
+        paths.map { |path| KnapsackPro::TestFileCleaner.clean(path) }
+      end
+
+      def schedule(paths)
+        @current_batch_examples = []
+        @batched_scheduled_paths << paths
+
+        paths.each do |path|
+          next unless KnapsackPro::Adapters::RSpecAdapter.id_path?(path)
+
+          file_path = KnapsackPro::Adapters::RSpecAdapter.parse_file_path(path)
+          @split_by_test_example_file_paths << KnapsackPro::TestFileCleaner.clean(file_path)
         end
       end
 
@@ -68,7 +88,7 @@ module KnapsackPro
           KnapsackPro::Adapters::RSpecAdapter.parse_file_path(example[:path])
         end
 
-        missing = (@scheduled_paths - recorded_paths).each_with_object({}) do |path, object|
+        missing = (@batched_scheduled_paths.flatten - recorded_paths).each_with_object({}) do |path, object|
           object[path] = { path: path, time_execution: 0.0 }
         end
 
@@ -87,12 +107,12 @@ module KnapsackPro
         now - @suite_started
       end
 
-      def unexecuted_test_files
+      def unexecuted_test_paths
         pending_paths = @paths.values
           .filter { |example| example[:time_execution] == 0.0 }
           .map { |example| example[:path] }
 
-        not_run_paths = @scheduled_paths -
+        not_run_paths = @batched_scheduled_paths.flatten -
           @paths.values
           .map { |example| example[:path] }
 
@@ -123,6 +143,8 @@ module KnapsackPro
         path = path_for(example)
         return if path.nil?
 
+        @current_batch_examples << example
+
         time_execution = time_execution_for(example, started_at)
         if accumulator.key?(path)
           accumulator[path][:time_execution] += time_execution

data/lib/knapsack_pro/formatters/time_tracker_fetcher.rb

@@ -10,10 +10,10 @@ module KnapsackPro
           .find { |f| f.class.to_s == "KnapsackPro::Formatters::TimeTracker" }
       end
 
-      def self.unexecuted_test_files
+      def self.unexecuted_test_paths
         time_tracker = call
         return [] unless time_tracker
-        time_tracker.unexecuted_test_files
+        time_tracker.unexecuted_test_paths
       end
     end
   end

data/lib/knapsack_pro/queue_allocator.rb

@@ -24,7 +24,15 @@ module KnapsackPro
       end
 
       def test_files
-        response.fetch('test_files')
+        if response.key?('test_files')
+          response.fetch('test_files')
+        else
+          response.fetch('paths').map { |path| { "path" => path, "time_execution" => nil } }
+        end
+      end
+
+      def id
+        response.fetch('batch_id', nil)
       end
 
       private
@@ -36,40 +44,38 @@ module KnapsackPro
       @test_suite = args.fetch(:test_suite)
       @ci_node_total = args.fetch(:ci_node_total)
       @ci_node_index = args.fetch(:ci_node_index)
-      @ci_node_build_id = args.fetch(:ci_node_build_id)
       @repository_adapter = args.fetch(:repository_adapter)
       @fallback_mode = false
+      @batch_index = -1
+      @batch_id = nil
     end
 
-    def test_file_paths(can_initialize_queue, executed_test_files, batch_uuid: SecureRandom.uuid)
+    def test_file_paths(can_initialize_queue, executed_test_files, batch_uuid: SecureRandom.uuid, time_tracker: nil)
+      @batch_index += 1
       return [] if @fallback_mode
 
-      batch = pull_tests_from_queue(can_initialize_queue, batch_uuid)
+      batch = pull_tests_from_queue(can_initialize_queue, batch_uuid, time_tracker: time_tracker)
 
       return switch_to_fallback_mode(executed_test_files: executed_test_files) if batch.connection_failed?
       return normalize_test_files(batch.test_files) if batch.queue_exists?
 
       test_files_result = test_suite.calculate_test_files
 
-      return try_initializing_queue(test_files_result.test_files, batch_uuid) if test_files_result.quick?
+      return try_initializing_queue(test_files_result.test_files, batch_uuid, time_tracker: time_tracker) if test_files_result.quick?
 
       # The tests to run were found slowly. By that time, the queue could have already been initialized by another CI node.
       # Attempt to pull tests from the queue to avoid the attempt to initialize the queue unnecessarily (queue initialization is an expensive request with a big test files payload).
-      batch = pull_tests_from_queue(can_initialize_queue, batch_uuid)
+      batch = pull_tests_from_queue(can_initialize_queue, batch_uuid, time_tracker: time_tracker)
 
       return switch_to_fallback_mode(executed_test_files: executed_test_files) if batch.connection_failed?
       return normalize_test_files(batch.test_files) if batch.queue_exists?
 
-      try_initializing_queue(test_files_result.test_files, batch_uuid)
+      try_initializing_queue(test_files_result.test_files, batch_uuid, time_tracker: time_tracker)
     end
 
     private
 
-    attr_reader :test_suite,
-      :ci_node_total,
-      :ci_node_index,
-      :ci_node_build_id,
-      :repository_adapter
+    attr_reader :test_suite, :ci_node_total, :ci_node_index, :repository_adapter
 
     def encrypted_branch
       KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch)
@@ -80,7 +86,7 @@ module KnapsackPro
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
 
-    def build_action(can_initialize_queue:, attempt_connect_to_queue:, batch_uuid:, test_files: nil)
+    def build_action_v1(can_initialize_queue:, attempt_connect_to_queue:, batch_uuid:, test_files: nil)
       if can_initialize_queue && !attempt_connect_to_queue
         raise 'Test files are required when initializing a new queue.' if test_files.nil?
         test_files = KnapsackPro::Crypto::Encryptor.call(test_files)
@@ -93,29 +99,89 @@ module KnapsackPro
         branch: encrypted_branch,
         node_total: ci_node_total,
         node_index: ci_node_index,
-        node_build_id: ci_node_build_id,
         test_files: test_files,
         batch_uuid: batch_uuid
       )
     end
 
-    def pull_tests_from_queue(can_initialize_queue, batch_uuid)
-      action = build_action(can_initialize_queue: can_initialize_queue, attempt_connect_to_queue: can_initialize_queue, batch_uuid: batch_uuid)
+    def build_action_v2(can_initialize_queue:, attempt_connect_to_queue:, time_tracker:, paths: nil)
+      if can_initialize_queue && !attempt_connect_to_queue
+        raise 'Test files are required when initializing a new queue.' if paths.nil?
+        paths = KnapsackPro::Crypto::Encryptor.paths(paths)
+      end
+
+      KnapsackPro::Client::API::V2::Queues.queue(
+        can_initialize_queue: can_initialize_queue,
+        attempt_connect_to_queue: attempt_connect_to_queue,
+        commit_hash: repository_adapter.commit_hash,
+        branch: encrypted_branch,
+        node_total: ci_node_total,
+        node_index: ci_node_index,
+        paths: paths,
+        failed_paths: time_tracker.current_batch_failed_paths,
+        batch_index: @batch_index,
+        batch_id: @batch_id
+      )
+    end
+
+    def pull_tests_from_queue(can_initialize_queue, batch_uuid, time_tracker: nil)
+      if time_tracker.nil?
+        pull_tests_from_queue_v1(can_initialize_queue, batch_uuid)
+      else
+        pull_tests_from_queue_v2(can_initialize_queue, time_tracker)
+      end
+    end
+
+    def pull_tests_from_queue_v1(can_initialize_queue, batch_uuid)
+      action = build_action_v1(can_initialize_queue: can_initialize_queue, attempt_connect_to_queue: can_initialize_queue, batch_uuid: batch_uuid)
       connection = KnapsackPro::Client::Connection.new(action)
       response = connection.call
       Batch.new(connection, response)
     end
 
-    def initialize_queue(tests_to_run, batch_uuid)
-      action = build_action(can_initialize_queue: true, attempt_connect_to_queue: false, batch_uuid: batch_uuid, test_files: tests_to_run)
+    def pull_tests_from_queue_v2(can_initialize_queue, time_tracker)
+      action = build_action_v2(can_initialize_queue: can_initialize_queue, attempt_connect_to_queue: can_initialize_queue, time_tracker: time_tracker)
+      connection = KnapsackPro::Client::Connection.new(action)
+      response = connection.call
+      Batch.new(connection, response).tap do |batch|
+        @batch_id = batch.id unless batch.connection_failed?
+      end
+    end
+
+    def initialize_queue_v1(tests_to_run, batch_uuid)
+      action = build_action_v1(can_initialize_queue: true, attempt_connect_to_queue: false, batch_uuid: batch_uuid, test_files: tests_to_run)
       connection = KnapsackPro::Client::Connection.new(action)
       response = connection.call
       Batch.new(connection, response)
     end
 
-    def try_initializing_queue(tests, batch_uuid)
-      result = initialize_queue(tests, batch_uuid)
+    def initialize_queue_v2(paths, time_tracker)
+      action = build_action_v2(can_initialize_queue: true, attempt_connect_to_queue: false, paths: paths, time_tracker: time_tracker)
+      connection = KnapsackPro::Client::Connection.new(action)
+      response = connection.call
+      Batch.new(connection, response).tap do |batch|
+        @batch_id = batch.id unless batch.connection_failed?
+      end
+    end
+
+    def try_initializing_queue(tests, batch_uuid, time_tracker: nil)
+      if time_tracker.nil?
+        try_initializing_queue_v1(tests, batch_uuid)
+      else
+        paths = tests.map { |test| test.fetch("path") }
+        try_initializing_queue_v2(paths, time_tracker)
+      end
+    end
+
+    def try_initializing_queue_v1(tests, batch_uuid)
+      result = initialize_queue_v1(tests, batch_uuid)
+      return switch_to_fallback_mode(executed_test_files: []) if result.connection_failed?
+
+      normalize_test_files(result.test_files)
+    end
 
+    def try_initializing_queue_v2(paths, time_tracker)
+      result = initialize_queue_v2(paths, time_tracker)
       return switch_to_fallback_mode(executed_test_files: []) if result.connection_failed?
 
       normalize_test_files(result.test_files)

data/lib/knapsack_pro/queue_allocator_builder.rb

@@ -7,8 +7,7 @@ module KnapsackPro
         test_suite: test_suite,
         ci_node_total: env.ci_node_total,
         ci_node_index: env.ci_node_index,
-        ci_node_build_id: env.ci_node_build_id,
-        repository_adapter: repository_adapter,
+        repository_adapter: repository_adapter
       )
     end
   end

data/lib/knapsack_pro/report.rb

@@ -42,8 +42,7 @@ module KnapsackPro
       end
 
       if test_files.empty?
-        KnapsackPro.logger.warn("No test files were executed on this CI node.")
-        KnapsackPro.logger.debug("This CI node likely started work late after the test files were already executed by other CI nodes consuming the queue.")
+        KnapsackPro.logger.info("No tests were executed because the test queue is empty.")
       end
 
       measured_test_files = test_files

data/lib/knapsack_pro/repository_adapters/git_adapter.rb

@@ -43,18 +43,39 @@ module KnapsackPro
       private
 
       def git_commit_authors
-        if KnapsackPro::Config::Env.ci? && shallow_repository?
-          command = 'git fetch --shallow-since "one month ago" --quiet 2>/dev/null'
-          begin
-            Timeout.timeout(5) do
-              `#{command}`
-            end
-          rescue Timeout::Error
-            KnapsackPro.logger.debug("Skip the `#{command}` command because it took too long.")
-          end
+        git_unshallow if KnapsackPro::Config::Env.ci? && shallow_repository?
+        `git log --since "one month ago" 2>/dev/null | git shortlog --summary --email 2>/dev/null`
+      end
+
+      def git_unshallow
+        args = ['git', 'fetch', '--quiet', '--shallow-since', 'one month ago']
+
+        begin
+          pid = Process.spawn(*args, [:out, :err] => File::NULL)
+        rescue StandardError => e
+          KnapsackPro.logger.debug("Failed to unshallow (#{args.join(' ')}): #{e.message}")
+          return
         end
 
-        `git log --since "one month ago" 2>/dev/null | git shortlog --summary --email 2>/dev/null`
+        begin
+          Timeout.timeout(5) { safe_waitpid(pid) }
+        rescue Timeout::Error
+          safe_kill(pid)
+          Timeout.timeout(1) { safe_waitpid(pid) } rescue Timeout::Error
+          KnapsackPro.logger.debug("Failed to unshallow (#{args.join(' ')}) in 5 seconds")
+        end
+      end
+
+      def safe_waitpid(pid)
+        Process.waitpid(pid)
+      rescue Errno::ECHILD
+        nil
+      end
+
+      def safe_kill(pid)
+        Process.kill('KILL', pid)
+      rescue Errno::ESRCH
+        nil
       end
 
       def git_build_author

data/lib/knapsack_pro/rspec/test_queue_initializer.rb

@@ -23,7 +23,7 @@ module KnapsackPro
       private
 
       def test_queue_exists?
-        action = KnapsackPro::Client::API::V1::Queues.connect
+        action = KnapsackPro::Client::API::V2::Queues.connect
         connection = KnapsackPro::Client::Connection.new(action)
         response = connection.call
 
@@ -53,7 +53,7 @@ module KnapsackPro
           exit 1
         end
 
-        action = KnapsackPro::Client::API::V1::Queues.initialize(paths)
+        action = KnapsackPro::Client::API::V2::Queues.initialize(paths)
         connection = KnapsackPro::Client::Connection.new(action)
         response = connection.call
 

data/lib/knapsack_pro/runners/queue/base_runner.rb

@@ -26,7 +26,8 @@ module KnapsackPro
         def test_file_paths(args)
           can_initialize_queue = args.fetch(:can_initialize_queue)
           executed_test_files = args.fetch(:executed_test_files)
-          allocator.test_file_paths(can_initialize_queue, executed_test_files)
+          time_tracker = args.fetch(:time_tracker, nil)
+          allocator.test_file_paths(can_initialize_queue, executed_test_files, time_tracker: time_tracker)
         end
 
         def test_dir

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -13,6 +13,7 @@ module KnapsackPro
           KnapsackPro::Extensions::RSpecExtension.setup!
 
           ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_rspec
+          ENV['KNAPSACK_PRO_NODE_UUID'] = SecureRandom.uuid
 
           rspec_pure = KnapsackPro::Pure::Queue::RSpecPure.new
 
@@ -64,7 +65,7 @@ module KnapsackPro
             KnapsackPro.logger.error("Exception message: #{exception.message}")
             KnapsackPro.logger.error("Exception backtrace: #{exception.backtrace&.join("\n")}")
 
-            message = @rspec_pure.exit_summary(unexecuted_test_files)
+            message = @rspec_pure.exit_summary(unexecuted_test_paths)
             KnapsackPro.logger.warn(message) if message
 
             exit_code = @rspec_pure.error_exit_code(@rspec_runner.knapsack__error_exit_code)
@@ -178,15 +179,14 @@ module KnapsackPro
         end
 
         def pull_tests_from_queue(can_initialize_queue: false)
+          time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
           test_file_paths = test_file_paths(
             can_initialize_queue: can_initialize_queue,
-            executed_test_files: @node_test_file_paths
+            executed_test_files: @node_test_file_paths,
+            time_tracker: time_tracker
           )
           @node_test_file_paths += test_file_paths
-
-          time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
-          time_tracker.scheduled_paths = @node_test_file_paths
-
+          time_tracker.schedule(test_file_paths)
           test_file_paths
         end
 
@@ -225,8 +225,8 @@ module KnapsackPro
           end
         end
 
-        def unexecuted_test_files
-          KnapsackPro::Formatters::TimeTrackerFetcher.unexecuted_test_files
+        def unexecuted_test_paths
+          KnapsackPro::Formatters::TimeTrackerFetcher.unexecuted_test_paths
         end
       end
     end

data/lib/knapsack_pro/test_suite.rb

@@ -21,7 +21,7 @@ module KnapsackPro
       end
 
       slow_id_paths = adapter_class.calculate_slow_id_paths
-      test_files = adapter_class.concat_paths(all_test_files_to_run, slow_id_paths)
+      test_files = adapter_class.concat_test_files(all_test_files_to_run, slow_id_paths)
       @result = Result.new(test_files, false)
     end
 

data/lib/knapsack_pro/urls.rb

@@ -18,7 +18,7 @@ module KnapsackPro
 
     INSTALLATION_GUIDE = "#{HOST}/perma/ruby/installation-guide"
 
-    KNAPSACK_PRO_CI_NODE_BUILD_ID= "#{HOST}/perma/ruby/knapsack-pro-ci-node-build-id"
+    KNAPSACK_PRO_CI_NODE_BUILD_ID = "#{HOST}/perma/ruby/knapsack-pro-ci-node-build-id"
 
     QUEUE_MODE__CONNECTION_ERROR_WITH_FALLBACK_ENABLED_FALSE = "#{HOST}/perma/ruby/queue-mode-connection-error-with-fallback-enabled-false"
 
@@ -35,5 +35,7 @@ module KnapsackPro
     SPLIT_BY_TEST_EXAMPLES = "#{HOST}/perma/ruby/split-by-test-examples"
 
     TEST_UNIT__TEST_FILE_PATH_DETECTION = "#{HOST}/perma/ruby/test-unit-test-file-path-detection"
+
+    KNAPSACK_PRO_TEST_QUEUE_ID = "#{HOST}/perma/ruby/knapsack-pro-test-queue-id"
   end
 end

data/lib/knapsack_pro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KnapsackPro
-  VERSION = '9.2.2'
+  VERSION = '10.0.0'
 end

data/lib/knapsack_pro.rb

@@ -33,7 +33,6 @@ require_relative 'knapsack_pro/client/api/action'
 require_relative 'knapsack_pro/client/api/v1/base'
 require_relative 'knapsack_pro/client/api/v1/build_distributions'
 require_relative 'knapsack_pro/client/api/v1/build_subsets'
-require_relative 'knapsack_pro/client/api/v1/queues'
 require_relative 'knapsack_pro/client/connection'
 require_relative 'knapsack_pro/repository_adapters/base_adapter'
 require_relative 'knapsack_pro/repository_adapters/env_adapter'
@@ -83,6 +82,20 @@ require_relative 'knapsack_pro/crypto/decryptor'
 require_relative 'knapsack_pro/crypto/digestor'
 require_relative 'knapsack_pro/pure/queue/rspec_pure'
 
+module KnapsackPro
+  module Client
+    module API
+      module V1
+        autoload(:Queues, 'knapsack_pro/client/api/v1/queues')
+      end
+
+      module V2
+        autoload(:Queues, 'knapsack_pro/client/api/v2/queues')
+      end
+    end
+  end
+end
+
 require 'knapsack_pro/railtie' if defined?(Rails::Railtie)
 
 module KnapsackPro

data/lib/tasks/queue/rspec.rake

@@ -20,6 +20,7 @@ namespace :knapsack_pro do
 
         ENV.delete('SPEC_OPTS') # Ignore `SPEC_OPTS` to not affect the RSpec execution within this rake task
         ENV['KNAPSACK_PRO_TEST_SUITE_TOKEN'] = KnapsackPro::Config::Env.test_suite_token_rspec
+        ENV['KNAPSACK_PRO_TEST_RUNNER'] = 'rspec'
 
         KnapsackPro::RSpec::TestQueueInitializer.new.call(args[:rspec_args].to_s)
       end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 9.2.2
+  version: 10.0.0
 platform: ruby
 authors:
 - ArturT
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -23,6 +23,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -93,20 +107,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rspec-its
   requirement: !ruby/object:Gem::Requirement
@@ -202,7 +202,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- bin/knapsack_pro
+- exe/knapsack_pro
 - lib/knapsack_pro.rb
 - lib/knapsack_pro/adapters/base_adapter.rb
 - lib/knapsack_pro/adapters/cucumber_adapter.rb
@@ -218,7 +218,10 @@ files:
 - lib/knapsack_pro/client/api/v1/build_distributions.rb
 - lib/knapsack_pro/client/api/v1/build_subsets.rb
 - lib/knapsack_pro/client/api/v1/queues.rb
+- lib/knapsack_pro/client/api/v2/queues.rb
 - lib/knapsack_pro/client/connection.rb
+- lib/knapsack_pro/commands.rb
+- lib/knapsack_pro/commands/retry_failed_tests.rb
 - lib/knapsack_pro/config/ci/app_veyor.rb
 - lib/knapsack_pro/config/ci/base.rb
 - lib/knapsack_pro/config/ci/buildkite.rb
@@ -303,6 +306,7 @@ metadata:
   documentation_uri: https://docs.knapsackpro.com/knapsack_pro-ruby/guide/
   homepage_uri: https://knapsackpro.com
   source_code_uri: https://github.com/KnapsackPro/knapsack_pro-ruby
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -317,7 +321,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Knapsack Pro splits tests across parallel CI nodes and ensures each parallel
   job finish work at a similar time.

data/bin/knapsack_pro

@@ -1,25 +0,0 @@
-#!/usr/bin/env ruby
-
-require_relative '../lib/knapsack_pro'
-
-runner = ARGV[0]
-arguments = ARGV[1]
-
-MAP = {
-  'rspec' => KnapsackPro::Runners::RSpecRunner,
-  'queue:rspec' => KnapsackPro::Runners::Queue::RSpecRunner,
-  'cucumber' => KnapsackPro::Runners::CucumberRunner,
-  'queue:cucumber' => KnapsackPro::Runners::Queue::CucumberRunner,
-  'minitest' => KnapsackPro::Runners::MinitestRunner,
-  'queue:minitest' => KnapsackPro::Runners::Queue::MinitestRunner,
-  'test_unit' => KnapsackPro::Runners::TestUnitRunner,
-  'spinach' => KnapsackPro::Runners::SpinachRunner,
-}
-
-runner_class = MAP[runner]
-
-if runner_class
-  runner_class.run(arguments)
-else
-  raise 'Undefined runner. Please provide runner name and optional arguments, for instance: knapsack_pro rspec "--color --profile"'
-end