knapsack_pro 11.0.0 → 11.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 228fc40233719e372291f82fe1c6d5c25ed0396ae12fb3e0f28edac71e69754d
-  data.tar.gz: ce7cb564e98830745734efd59850e50c43c0abd49ed6131ecd4392801dd4758e
+  metadata.gz: 4048ec74df7fa6a88b8f44df5afe764f841c2f8823ae08964e03c6ae87e5639c
+  data.tar.gz: 312475082e64ade4db5a8b32160b9ddb294878140149140f1eb6921e08c4d242
 SHA512:
-  metadata.gz: ff6be1076a78549e3f4a8f57362ec560cd482ddd332513280bf67155248e7066478291014cd6f166c904344be81d520f374fdb565884d4540f2344b8577739d1
-  data.tar.gz: 8131b135fa3f86574b77ae3576ed538c88c871073c1d9756f283ad88245a84ed50dd55ab203ab330a39e578e5aeedf3e700051f8bab3f622611db9380926e42a
+  metadata.gz: 4c85eb9aa82c714abf8bbbd609725e8f9f05f7c862fd46daf75cb4db4f52ea45e50d90fe731669af1bb592380ed1644441ed73362f90a6f6468a477edbbbf32f
+  data.tar.gz: a8cc23ab19ffb5a4c193942e3ae712c3ff28904098a246583389dafc691f95ed61b4c04509aa9dc6b6115d0470a30a09a45c61000c0f3a609e7c3142e4739555

data/lib/knapsack_pro/adapters/cucumber_adapter.rb

@@ -6,28 +6,7 @@ module KnapsackPro
       TEST_DIR_PATTERN = 'features/**{,/*/**}/*.feature'
 
       def self.test_path(object)
-        if ::Cucumber::VERSION.to_i >= 2
-          test_case = object
-          test_case.location.file
-        else
-          if object.respond_to?(:scenario_outline)
-            if object.scenario_outline.respond_to?(:feature)
-              # Cucumber < 1.3
-              object.scenario_outline.feature.file
-            else
-              # Cucumber >= 1.3
-              object.scenario_outline.file
-            end
-          else
-            if object.respond_to?(:feature)
-              # Cucumber < 1.3
-              object.feature.file
-            else
-              # Cucumber >= 1.3
-              object.file
-            end
-          end
-        end
+        object.location.file
       end
 
       def bind_time_tracker
@@ -50,7 +29,7 @@ module KnapsackPro
           exit_status = latest_error.status if latest_error.is_a?(SystemExit)
           # saving report makes API call which changes exit status
           # from cucumber so we need to preserve cucumber exit status
-          KnapsackPro::Report.save
+          KnapsackPro::Report.save(KnapsackPro.tracker.to_a, KnapsackPro.tracker.current_batch_failed_paths)
           ::Kernel.exit exit_status if exit_status
         end
       end
@@ -75,11 +54,7 @@ module KnapsackPro
       private
 
       def Around(*tag_expressions, &proc)
-        if ::Cucumber::VERSION.to_i >= 3
-          ::Cucumber::Glue::Dsl.register_rb_hook('around', tag_expressions, proc)
-        else
-          ::Cucumber::RbSupport::RbDsl.register_rb_hook('around', tag_expressions, proc)
-        end
+        ::Cucumber::Glue::Dsl.register_rb_hook('around', tag_expressions, proc)
       end
     end
   end

data/lib/knapsack_pro/adapters/minitest_adapter.rb

@@ -23,29 +23,33 @@ module KnapsackPro
       end
 
       module BindTimeTrackerMinitestPlugin
-        def before_setup
-          super
-          KnapsackPro.tracker.current_test_path = KnapsackPro::Adapters::MinitestAdapter.test_path(self)
-          KnapsackPro.tracker.start_timer
-        end
+        def run
+          path = KnapsackPro::Adapters::MinitestAdapter.test_path(self)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          result = super
+          elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+
+          failed = result.failures.any? { |failure| !failure.is_a?(::Minitest::Skip) }
+          KnapsackPro.tracker.stage(path, elapsed, failed: failed)
 
-        def after_teardown
-          KnapsackPro.tracker.stop_timer
-          super
+          result
         end
       end
 
       def bind_time_tracker
-        ::Minitest::Test.send(:include, BindTimeTrackerMinitestPlugin)
+        ::Minitest::Test.send(:prepend, BindTimeTrackerMinitestPlugin)
 
         add_post_run_callback do
+          KnapsackPro.tracker.flush_staged_tests
           KnapsackPro.logger.debug(KnapsackPro::Presenter.global_time)
         end
       end
 
       def bind_save_report
         add_post_run_callback do
-          KnapsackPro::Report.save
+          KnapsackPro.tracker.flush_staged_tests
+          KnapsackPro::Report.save(KnapsackPro.tracker.to_a, KnapsackPro.tracker.current_batch_failed_paths)
         end
       end
 
@@ -55,29 +59,30 @@ module KnapsackPro
       end
 
       module BindNormalModeMinitestPlugin
-        def before_setup
-          super
-
+        def run
           unless ENV['KNAPSACK_PRO_BEFORE_QUEUE_HOOK_CALLED']
             KnapsackPro::Hooks::Queue.call_before_queue
             ENV['KNAPSACK_PRO_BEFORE_QUEUE_HOOK_CALLED'] = 'true'
           end
 
-          KnapsackPro.tracker.current_test_path = KnapsackPro::Adapters::MinitestAdapter.test_path(self)
-          KnapsackPro.tracker.start_timer
-        end
+          path = KnapsackPro::Adapters::MinitestAdapter.test_path(self)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+          result = super
+          elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
 
-        def after_teardown
-          KnapsackPro.tracker.stop_timer
+          failed = result.failures.any? { |failure| !failure.is_a?(::Minitest::Skip) }
+          KnapsackPro.tracker.stage(path, elapsed, failed: failed)
 
-          super
+          result
         end
       end
 
       def bind_normal_mode
-        ::Minitest::Test.send(:include, BindNormalModeMinitestPlugin)
+        ::Minitest::Test.send(:prepend, BindNormalModeMinitestPlugin)
 
         add_post_run_callback do
+          KnapsackPro.tracker.flush_staged_tests
           KnapsackPro.logger.debug(KnapsackPro::Presenter.global_time)
         end
       end

data/lib/knapsack_pro/adapters/rspec_adapter.rb

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
 
-require_relative '../formatters/time_tracker_fetcher'
+require_relative '../formatters/rspec/time_tracker_fetcher'
 
 module KnapsackPro
   module Adapters
     class RSpecAdapter < BaseAdapter
       TEST_DIR_PATTERN = 'spec/**{,/*/**}/*_spec.rb'
       # https://github.com/rspec/rspec/blob/86b5e4218eece4c1913fe9aad24c0a96d8bc9f40/rspec-core/lib/rspec/core/example.rb#L122
-      REGEX = /\A(.*?)(?:\[([\d\s:,]+)\])?\z/.freeze
+      ID_REGEX = /\[[\d\s:,]+\]\z/.freeze
+      ID_PATH_PREFIX = '['
+      ID_PATH_SUFFIX = ']'
 
       def self.split_by_test_cases_enabled?
         return false unless KnapsackPro::Config::Env.rspec_split_by_test_examples?
@@ -57,28 +59,33 @@ module KnapsackPro
       end
 
       def self.file_path_for(example)
-        [
-          -> { parse_file_path(example.id) },
-          -> { example.metadata[:file_path] },
-          -> { example.metadata[:example_group][:file_path] },
-          -> { top_level_group(example)[:file_path] },
-        ]
-          .each do |path|
-            p = path.call
-            return p if p.include?('_spec.rb') || p.include?('.feature')
-          end
+        p = parse_file_path(example.id)
+        return p if p.include?('_spec.rb') || p.include?('.feature')
+
+        p = example.metadata[:file_path]
+        return p if p.include?('_spec.rb') || p.include?('.feature')
+
+        p = example.metadata[:example_group][:file_path]
+        return p if p.include?('_spec.rb') || p.include?('.feature')
 
-        return ''
+        p = top_level_group(example)[:file_path]
+        return p if p.include?('_spec.rb') || p.include?('.feature')
+
+        ''
       end
 
       def self.parse_file_path(path)
-        file, _id = path.match(REGEX).captures
-        file
+        return path unless path.end_with?(ID_PATH_SUFFIX)
+
+        index = path.rindex(ID_PATH_PREFIX)
+        return path if index.nil?
+        return path unless ID_REGEX.match?(path, index)
+
+        path[0, index]
       end
 
       def self.id_path?(path)
-        _file, id = path.match(REGEX).captures
-        !id.nil?
+        path.end_with?(ID_PATH_SUFFIX) && ID_REGEX.match?(path)
       end
 
       def self.concat_test_files(test_files, id_paths)
@@ -129,7 +136,7 @@ module KnapsackPro
       def log_tests_duration
         ::RSpec.configure do |config|
           config.append_after(:suite) do
-            time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+            time_tracker = KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.call
             formatted = KnapsackPro::Presenter.global_time(time_tracker.duration)
             KnapsackPro.logger.debug(formatted)
           end
@@ -141,7 +148,7 @@ module KnapsackPro
 
         ::RSpec.configure do |config|
           config.append_before(:suite) do
-            time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+            time_tracker = KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.call
             time_tracker.schedule(KnapsackPro::Adapters::RSpecAdapter.scheduled_paths)
           end
         end
@@ -150,8 +157,8 @@ module KnapsackPro
       def bind_save_report
         ::RSpec.configure do |config|
           config.after(:suite) do
-            time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
-            KnapsackPro::Report.save(time_tracker.batch)
+            time_tracker = KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.call
+            KnapsackPro::Report.save(time_tracker.batch, time_tracker.current_batch_failed_paths)
           end
         end
       end

data/lib/knapsack_pro/formatters/cucumber/failures_tracker.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  module Formatters
+    module Cucumber
+      class FailuresTracker
+        def initialize(config)
+          failed_locations = {}
+
+          config.on_event :test_case_finished do |event|
+            location = event.test_case.location
+            id_path = "#{location.file}:#{location.line}"
+
+            ok =
+              if config.respond_to?(:strict)
+                event.result.ok?(strict: config.strict)
+              else
+                event.result.ok?
+              end
+
+            if ok
+              failed_locations.delete(id_path)
+            else
+              failed_locations[id_path] = location
+            end
+          end
+
+          config.on_event :test_run_finished do |_event|
+            paths = failed_locations.map do |id_path, location|
+              # TODO: Tracker currently only tracks by file_path. Track id_path and use the following:
+              # KnapsackPro::Config::Env.test_files_encrypted? ? location.file : id_path
+              location.file
+            end
+
+            paths.uniq.each do |path|
+              KnapsackPro.tracker.mark_failed(path)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/knapsack_pro/formatters/rspec/time_tracker.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'set'
+require_relative '../../utils'
+
+module KnapsackPro
+  module Formatters
+    module RSpec
+      class TimeTracker
+        ::RSpec::Core::Formatters.register self,
+          :example_group_started,
+          :example_started,
+          :example_finished,
+          :example_group_finished
+
+        attr_reader :output # RSpec < v3.10.2
+
+        def initialize(_output)
+          @output = StringIO.new
+          @time_each = nil
+          @time_all = nil
+          @time_all_by_group_id_path = Hash.new(0)
+          @group = {}
+          @paths = {}
+          @suite_started = now
+          @batched_scheduled_paths = []
+          @split_by_test_example_file_paths = Set.new
+          @current_batch_examples = []
+        end
+
+        def current_batch_failed_paths
+          examples_by_file_path = @current_batch_examples
+            .group_by { |example| file_path_for(example) }
+
+          paths =
+            examples_by_file_path.flat_map do |file_path, examples|
+              failed_id_paths = examples.filter { |example| example.execution_result.status.to_s == "failed" }.map(&:id)
+              next [] if failed_id_paths.none?
+              next file_path if KnapsackPro::Config::Env.test_files_encrypted?
+              # Other nodes may have run some examples from this file, it's not safe to compact.
+              next failed_id_paths if rspec_split_by_test_example?(file_path)
+              next file_path if failed_id_paths.size == examples.size
+
+              failed_id_paths
+            end
+
+          paths.map { |path| KnapsackPro::TestFileCleaner.clean(path) }
+        end
+
+        def schedule(paths)
+          @current_batch_examples = []
+          @batched_scheduled_paths << paths
+
+          paths.each do |path|
+            next unless KnapsackPro::Adapters::RSpecAdapter.id_path?(path)
+
+            file_path = KnapsackPro::Adapters::RSpecAdapter.parse_file_path(path)
+            @split_by_test_example_file_paths << KnapsackPro::TestFileCleaner.clean(file_path)
+          end
+        end
+
+        def example_group_started(notification)
+          record_time_all(notification.group.parent_groups[1], @time_all_by_group_id_path, @time_all)
+          @time_all = now
+        end
+
+        def example_started(notification)
+          record_time_all(notification.example.example_group, @time_all_by_group_id_path, @time_all)
+          @time_each = now
+        end
+
+        def example_finished(notification)
+          record_example(@group, notification.example, @time_each)
+          @time_all = now
+        end
+
+        def example_group_finished(notification)
+          record_time_all(notification.group, @time_all_by_group_id_path, @time_all)
+          @time_all = now
+          return unless top_level_group?(notification.group)
+
+          add_hooks_time(@group, @time_all_by_group_id_path)
+          @time_all_by_group_id_path = Hash.new(0)
+          merge!(@paths, @group)
+          @group = {}
+        end
+
+        def queue
+          recorded_paths = @paths.values.map do |example|
+            KnapsackPro::Adapters::RSpecAdapter.parse_file_path(example[:path])
+          end
+
+          missing = (@batched_scheduled_paths.flatten - recorded_paths).each_with_object({}) do |path, object|
+            object[path] = { path: path, time_execution: 0.0 }
+          end
+
+          merge!(@paths, missing)
+
+          @paths.values.map do |example|
+            example.transform_keys(&:to_s)
+          end
+        end
+
+        def batch
+          @paths.values.map do |example|
+            example.transform_keys(&:to_s)
+          end
+        end
+
+        def duration
+          now - @suite_started
+        end
+
+        def unexecuted_test_paths
+          pending_paths = @paths.values
+            .filter { |example| example[:time_execution] == 0.0 }
+            .map { |example| example[:path] }
+
+          not_run_paths = @batched_scheduled_paths.flatten -
+            @paths.values
+            .map { |example| example[:path] }
+
+          pending_paths + not_run_paths
+        end
+
+        private
+
+        def top_level_group?(group)
+          group.metadata[:parent_example_group].nil?
+        end
+
+        def add_hooks_time(group, time_all_by_group_id_path)
+          group.each do |_, example|
+            next if example[:time_execution] == 0.0
+
+            example[:time_execution] += time_all_by_group_id_path.reduce(0.0) do |sum, (group_id_path, time)|
+              # :path is a file path (a_spec.rb), sum any before/after(:all) in the file
+              next sum + time if group_id_path.start_with?(example[:path])
+              # :path is an id path (a_spec.rb[1:1]), sum any before/after(:all) above it
+              next sum + time if example[:path].start_with?(group_id_path[0..-2])
+              sum
+            end
+          end
+        end
+
+        def record_example(accumulator, example, started_at)
+          path = path_for(example)
+          return if path.nil?
+
+          @current_batch_examples << example
+
+          time_execution = time_execution_for(example, started_at)
+          if accumulator.key?(path)
+            accumulator[path][:time_execution] += time_execution
+          else
+            accumulator[path] = { path: path, time_execution: time_execution }
+          end
+        end
+
+        def record_time_all(group, time_all_by_group_id_path, time_all)
+          return unless group # above top level group
+
+          group_id_path = KnapsackPro::TestFileCleaner.clean(group.id)
+          time_all_by_group_id_path[group_id_path] += now - time_all
+        end
+
+        def path_for(example)
+          file_path = file_path_for(example)
+          return nil if file_path == ""
+
+          if rspec_split_by_test_example?(file_path)
+            KnapsackPro::TestFileCleaner.clean(example.id)
+          else
+            file_path
+          end
+        end
+
+        def rspec_split_by_test_example?(file_path)
+          @split_by_test_example_file_paths.include?(file_path)
+        end
+
+        def file_path_for(example)
+          KnapsackPro::TestFileCleaner.clean(KnapsackPro::Adapters::RSpecAdapter.file_path_for(example))
+        end
+
+        def time_execution_for(example, started_at)
+          if example.execution_result.status.to_s == "pending"
+            0.0
+          else
+            (now - started_at).to_f
+          end
+        end
+
+        def merge!(h1, h2)
+          h2.each do |path, example|
+            if recorded = h1[path]
+              recorded[:time_execution] += example[:time_execution]
+            else
+              h1[path] = example
+            end
+          end
+        end
+
+        def now
+          KnapsackPro::Utils.time_now
+        end
+      end
+    end
+  end
+end

data/lib/knapsack_pro/formatters/rspec/time_tracker_fetcher.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  module Formatters
+    module RSpec
+      class TimeTrackerFetcher
+        def self.call
+          ::RSpec
+            .configuration
+            .formatters
+            .find { |f| f.class.to_s == "KnapsackPro::Formatters::RSpec::TimeTracker" }
+        end
+
+        def self.unexecuted_test_paths
+          time_tracker = call
+          return [] unless time_tracker
+          time_tracker.unexecuted_test_paths
+        end
+      end
+    end
+  end
+end

data/lib/knapsack_pro/pure/queue/rspec_pure.rb

@@ -6,7 +6,7 @@ module KnapsackPro
       class RSpecPure
         FAILURE_EXIT_CODE = 1
         FORMATTERS = [
-          'KnapsackPro::Formatters::TimeTracker',
+          'KnapsackPro::Formatters::RSpec::TimeTracker',
         ]
 
         def add_knapsack_pro_formatters_to(spec_opts)

data/lib/knapsack_pro/queue_allocator.rb

@@ -51,27 +51,27 @@ module KnapsackPro
       @current_batch = nil
     end
 
-    def test_file_paths(can_initialize_queue, executed_test_files, time_tracker: nil)
+    def test_file_paths(can_initialize_queue, executed_test_files, failed_paths)
       @batch_index += 1
       return [] if @fallback_mode
 
-      batch = pull_tests_from_queue(can_initialize_queue, time_tracker: time_tracker)
+      batch = pull_tests_from_queue(can_initialize_queue, failed_paths)
 
       return switch_to_fallback_mode(executed_test_files: executed_test_files) if batch.connection_failed?
       return normalize_test_files(batch.test_files) if batch.queue_exists?
 
       test_files_result = test_suite.calculate_test_files
 
-      return try_initializing_queue(test_files_result.test_files.map { |test| test.fetch("path") }, time_tracker: time_tracker) if test_files_result.quick?
+      return try_initializing_queue(test_files_result.test_files.map { |test| test.fetch("path") }, failed_paths) if test_files_result.quick?
 
       # The tests to run were found slowly. By that time, the queue could have already been initialized by another CI node.
       # Attempt to pull tests from the queue to avoid the attempt to initialize the queue unnecessarily (queue initialization is an expensive request with a big test files payload).
-      batch = pull_tests_from_queue(can_initialize_queue, time_tracker: time_tracker)
+      batch = pull_tests_from_queue(can_initialize_queue, failed_paths)
 
       return switch_to_fallback_mode(executed_test_files: executed_test_files) if batch.connection_failed?
       return normalize_test_files(batch.test_files) if batch.queue_exists?
 
-      try_initializing_queue(test_files_result.test_files.map { |test| test.fetch("path") }, time_tracker: time_tracker)
+      try_initializing_queue(test_files_result.test_files.map { |test| test.fetch("path") }, failed_paths)
     end
 
     private
@@ -87,16 +87,15 @@ module KnapsackPro
       KnapsackPro::TestFilePresenter.paths(decrypted_test_files)
     end
 
-    # time_tracker is nil for non-RSpec queue runners
-    def build_action(can_initialize_queue:, attempt_connect_to_queue:, time_tracker:, paths: nil)
+    def build_action(can_initialize_queue:, attempt_connect_to_queue:, failed_paths:, paths: nil)
       if can_initialize_queue && !attempt_connect_to_queue
         raise 'Test files are required when initializing a new queue.' if paths.nil?
         paths = KnapsackPro::Crypto::Encryptor.paths(paths)
       end
 
       failed_paths =
-        if time_tracker
-          KnapsackPro::Crypto::Encryptor.paths(time_tracker.current_batch_failed_paths)
+        if failed_paths
+          KnapsackPro::Crypto::Encryptor.paths(failed_paths)
         elsif @batch_index.positive?
           KnapsackPro::Crypto::Encryptor.paths(@current_batch.paths)
         end
@@ -116,22 +115,22 @@ module KnapsackPro
       )
     end
 
-    def pull_tests_from_queue(can_initialize_queue, time_tracker: nil)
-      action = build_action(can_initialize_queue: can_initialize_queue, attempt_connect_to_queue: can_initialize_queue, time_tracker: time_tracker)
+    def pull_tests_from_queue(can_initialize_queue, failed_paths)
+      action = build_action(can_initialize_queue: can_initialize_queue, attempt_connect_to_queue: can_initialize_queue, failed_paths: failed_paths)
       connection = KnapsackPro::Client::Connection.new(action)
       response = connection.call
       @current_batch = Batch.new(connection, response)
     end
 
-    def initialize_queue(paths, time_tracker: nil)
-      action = build_action(can_initialize_queue: true, attempt_connect_to_queue: false, paths: paths, time_tracker: time_tracker)
+    def initialize_queue(paths, failed_paths)
+      action = build_action(can_initialize_queue: true, attempt_connect_to_queue: false, paths: paths, failed_paths: failed_paths)
       connection = KnapsackPro::Client::Connection.new(action)
       response = connection.call
       @current_batch = Batch.new(connection, response)
     end
 
-    def try_initializing_queue(paths, time_tracker: nil)
-      result = initialize_queue(paths, time_tracker: time_tracker)
+    def try_initializing_queue(paths, failed_paths)
+      result = initialize_queue(paths, failed_paths)
       return switch_to_fallback_mode(executed_test_files: []) if result.connection_failed?
 
       normalize_test_files(result.test_files)

data/lib/knapsack_pro/report.rb

@@ -2,7 +2,7 @@
 
 module KnapsackPro
   class Report
-    def self.save(test_files = nil)
+    def self.save(test_files = nil, failed_paths = nil)
       if test_files.nil?
         test_files = KnapsackPro.tracker.to_a
       end
@@ -11,31 +11,39 @@ module KnapsackPro
         KnapsackPro.logger.info("No tests were executed because the test queue is empty.")
       end
 
-      acknowledge_single_batch(test_files)
+      acknowledge_single_batch(test_files, failed_paths)
       create_build_subset(test_files)
     end
 
     def self.save_subset_queue_to_file
       test_files = KnapsackPro.tracker.to_a
-
       subset_queue_id = KnapsackPro::Config::Env.subset_queue_id
-
       KnapsackPro::Config::TempFiles.ensure_temp_directory_exists!
       FileUtils.mkdir_p(queue_path)
-
       subset_queue_file_name = "#{subset_queue_id}.json"
       report_path = File.join(queue_path, subset_queue_file_name)
       report_json = JSON.pretty_generate(test_files)
-
       File.open(report_path, 'w+') do |f|
         f.write(report_json)
       end
+
+      failed_paths = KnapsackPro.tracker.current_batch_failed_paths
+      json = JSON.pretty_generate(failed_paths)
+      File.open(current_batch_failed_paths_path, 'w+') do |f|
+        f.write(json)
+      end
+    end
+
+    def self.current_batch_failed_paths
+      JSON.parse(File.read(current_batch_failed_paths_path))
     end
 
     def self.save_node_queue_to_api(test_files = nil)
       if test_files.nil?
         test_files = []
         Dir.glob("#{queue_path}/*.json").each do |file|
+          next if file == current_batch_failed_paths_path
+
           report = JSON.parse(File.read(file))
           test_files += report
         end
@@ -59,11 +67,10 @@ module KnapsackPro
       create_build_subset(test_files)
     end
 
-    # TODO: Add support for Retry-only-Failures
-    def self.acknowledge_single_batch(test_files)
+    def self.acknowledge_single_batch(test_files, failed_paths)
       repository_adapter = KnapsackPro::RepositoryAdapterInitiator.call
       paths = KnapsackPro::Utils.unsymbolize(test_files).map { |test_file| test_file.fetch('path') }
-      failed_paths = KnapsackPro::Crypto::Encryptor.paths(paths)
+      failed_paths = paths if failed_paths.nil?
       action = KnapsackPro::Client::API::V2::Queues.queue(
         attempt_connect_to_queue: false,
         batch_id: ENV['KNAPSACK_PRO_BATCH_ID'], # nil in Fallback Mode
@@ -71,7 +78,7 @@ module KnapsackPro
         branch: KnapsackPro::Crypto::BranchEncryptor.call(repository_adapter.branch),
         can_initialize_queue: false,
         commit_hash: repository_adapter.commit_hash,
-        failed_paths: failed_paths,
+        failed_paths: KnapsackPro::Crypto::Encryptor.paths(failed_paths),
         node_total: KnapsackPro::Config::Env.ci_node_total,
         node_index: KnapsackPro::Config::Env.ci_node_index,
         node_uuid: ENV.fetch('KNAPSACK_PRO_NODE_UUID'),
@@ -115,5 +122,9 @@ module KnapsackPro
     def self.queue_path
       "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/queue/#{KnapsackPro::Config::Env.queue_id}"
     end
+
+    def self.current_batch_failed_paths_path
+      File.join(KnapsackPro::Report.queue_path, "current_batch_failed_paths.json")
+    end
   end
 end

data/lib/knapsack_pro/runners/base_runner.rb

@@ -26,7 +26,7 @@ module KnapsackPro
 
       def test_files_to_execute_exist?
         if test_file_paths.empty?
-          KnapsackPro.logger.info("Knapsack Pro API returned no test files to execute for the node this time. The reason might be that you changed recently a number of total nodes or you removed some test files. Please create a new commit to get a better test suite split next time.")
+          KnapsackPro.logger.info("No tests were executed because the test queue is empty.")
           false
         else
           true

data/lib/knapsack_pro/runners/cucumber_runner.rb

@@ -24,8 +24,19 @@ module KnapsackPro
             Rake::Task[task_name].clear
           end
 
+          require "cucumber/cli/main"
+          require "shellwords"
+          formats = Cucumber::Cli::Main.new(Shellwords.split(args || "")).configuration.formats
+          formats << ["KnapsackPro::Formatters::Cucumber::FailuresTracker", {}, File::NULL]
+          formats_arg = formats.flat_map do |name, options, output|
+            format = [name, *options.map { |key, value| "#{key}=#{value}" }].join(",")
+            formats = ["--format", format]
+            formats.concat(["--out", output]) if output.is_a?(String)
+            formats
+          end
+
           ::Cucumber::Rake::Task.new(task_name) do |t|
-            t.cucumber_opts = "#{args} --require #{runner.test_dir} -- #{runner.stringify_test_file_paths}"
+            t.cucumber_opts = "#{args} --require #{runner.test_dir} #{formats_arg.shelljoin} -- #{runner.stringify_test_file_paths}"
           end
           Rake::Task[task_name].invoke
         end

data/lib/knapsack_pro/runners/queue/base_runner.rb

@@ -25,8 +25,8 @@ module KnapsackPro
         def test_file_paths(args)
           can_initialize_queue = args.fetch(:can_initialize_queue)
           executed_test_files = args.fetch(:executed_test_files)
-          time_tracker = args.fetch(:time_tracker, nil)
-          allocator.test_file_paths(can_initialize_queue, executed_test_files, time_tracker: time_tracker)
+          failed_paths = args.fetch(:failed_paths)
+          allocator.test_file_paths(can_initialize_queue, executed_test_files, failed_paths)
         end
 
         def test_dir

data/lib/knapsack_pro/runners/queue/cucumber_runner.rb

@@ -21,7 +21,7 @@ module KnapsackPro
             can_initialize_queue: true,
             args: args,
             exitstatus: 0,
-            node_test_file_paths: [],
+            node_test_file_paths: []
           }
           while accumulator[:status] == :next
             handle_signal!
@@ -43,7 +43,8 @@ module KnapsackPro
 
           test_file_paths = runner.test_file_paths(
             can_initialize_queue: can_initialize_queue,
-            executed_test_files: node_test_file_paths
+            executed_test_files: node_test_file_paths,
+            failed_paths: can_initialize_queue ? [] : KnapsackPro::Report.current_batch_failed_paths
           )
 
           if test_file_paths.empty?
@@ -83,7 +84,7 @@ module KnapsackPro
               can_initialize_queue: false,
               args: args,
               exitstatus: exitstatus,
-              node_test_file_paths: node_test_file_paths,
+              node_test_file_paths: node_test_file_paths
             }
           end
         end
@@ -91,11 +92,22 @@ module KnapsackPro
         private
 
         def self.cucumber_run(runner, test_file_paths, args)
+          require "cucumber/cli/main"
+          require "shellwords"
+          formats = Cucumber::Cli::Main.new(Shellwords.split(args || "")).configuration.formats
+          formats << ["KnapsackPro::Formatters::Cucumber::FailuresTracker", {}, File::NULL]
+          formats_arg = formats.flat_map do |name, options, output|
+            format = [name, *options.map { |key, value| "#{key}=#{value}" }].join(",")
+            formats = ["--format", format]
+            formats.concat(["--out", output]) if output.is_a?(String)
+            formats
+          end
+
           stringify_test_file_paths = KnapsackPro::TestFilePresenter.stringify_paths(test_file_paths)
 
           cmd = [
             KnapsackPro::Config::Env.cucumber_queue_prefix,
-            %Q[cucumber #{args} --require #{runner.test_dir} -- #{stringify_test_file_paths}]
+            %Q[cucumber #{args} --require #{runner.test_dir} #{formats_arg.shelljoin} -- #{stringify_test_file_paths}]
           ].join(' ')
 
           Kernel.system(cmd)

data/lib/knapsack_pro/runners/queue/minitest_runner.rb

@@ -54,7 +54,8 @@ module KnapsackPro
 
           test_file_paths = runner.test_file_paths(
             can_initialize_queue: can_initialize_queue,
-            executed_test_files: node_test_file_paths
+            executed_test_files: node_test_file_paths,
+            failed_paths: can_initialize_queue ? [] : KnapsackPro::Report.current_batch_failed_paths
           )
 
           if test_file_paths.empty?
@@ -86,6 +87,7 @@ module KnapsackPro
 
             KnapsackPro::Hooks::Queue.call_after_subset_queue
 
+            KnapsackPro.tracker.flush_staged_tests
             KnapsackPro::Report.save_subset_queue_to_file
 
             return {

data/lib/knapsack_pro/runners/queue/rspec_runner.rb

@@ -7,8 +7,8 @@ module KnapsackPro
         def self.run(args, stream_error = $stderr, stream_out = $stdout)
           require 'rspec/core'
           require_relative '../../extensions/rspec_extension'
-          require_relative '../../formatters/time_tracker'
-          require_relative '../../formatters/time_tracker_fetcher'
+          require_relative '../../formatters/rspec/time_tracker'
+          require_relative '../../formatters/rspec/time_tracker_fetcher'
 
           KnapsackPro::Extensions::RSpecExtension.setup!
 
@@ -163,7 +163,7 @@ module KnapsackPro
 
           log_rspec_queue_command
 
-          time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+          time_tracker = KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.call
           KnapsackPro::Report.save_node_queue_to_api(time_tracker&.queue)
 
           Kernel.exit(exit_code)
@@ -178,11 +178,11 @@ module KnapsackPro
         end
 
         def pull_tests_from_queue(can_initialize_queue: false)
-          time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+          time_tracker = KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.call
           test_file_paths = test_file_paths(
             can_initialize_queue: can_initialize_queue,
             executed_test_files: @node_test_file_paths,
-            time_tracker: time_tracker
+            failed_paths: time_tracker.current_batch_failed_paths
           )
           @node_test_file_paths += test_file_paths
           time_tracker.schedule(test_file_paths)
@@ -225,7 +225,7 @@ module KnapsackPro
         end
 
         def unexecuted_test_paths
-          KnapsackPro::Formatters::TimeTrackerFetcher.unexecuted_test_paths
+          KnapsackPro::Formatters::RSpec::TimeTrackerFetcher.unexecuted_test_paths
         end
       end
     end

data/lib/knapsack_pro/runners/rspec_runner.rb

@@ -39,7 +39,7 @@ module KnapsackPro
 
       # Use RSpec::Core::ConfigurationOptions to respect external configurations like .rspec
       def self.formatters
-        require_relative '../formatters/time_tracker'
+        require_relative '../formatters/rspec/time_tracker'
 
         formatters = ::RSpec::Core::ConfigurationOptions
           .new([])
@@ -51,7 +51,7 @@ module KnapsackPro
             arg
           end
         formatters = ['--format progress'] if formatters.empty?
-        formatters += ["--format #{KnapsackPro::Formatters::TimeTracker}"]
+        formatters += ["--format #{KnapsackPro::Formatters::RSpec::TimeTracker}"]
         formatters.join(' ')
       end
     end

data/lib/knapsack_pro/tracker.rb

@@ -6,7 +6,7 @@ module KnapsackPro
 
     DEFAULT_TEST_FILE_TIME = 0.0 # seconds
 
-    attr_reader :global_time, :test_files_with_time, :prerun_tests_loaded
+    attr_reader :global_time, :test_files_with_time, :prerun_tests_loaded, :current_batch_failed_paths
     attr_writer :current_test_path
 
     def initialize
@@ -74,20 +74,48 @@ module KnapsackPro
       load_prerun_tests unless prerun_tests_loaded
 
       test_files = []
+
       @test_files_with_time.each do |path, hash|
         test_files << {
           path: path,
           time_execution: hash[:time_execution]
         }
       end
+
       test_files
     end
 
+    def mark_failed(path)
+      @current_batch_failed_paths << KnapsackPro::TestFileCleaner.clean(path)
+    end
+
+    # Thread-safe: first stage then flush_staged_tests
+    def stage(path, execution_time, failed: false)
+      @staged_tests << [KnapsackPro::TestFileCleaner.clean(path), execution_time, failed]
+    end
+
+    # Thread-safe: first stage then flush_staged_tests
+    def flush_staged_tests
+      loop do
+        begin
+          path, execution_time, failed = @staged_tests.pop(true)
+        rescue ThreadError
+          break
+        end
+
+        update_global_time(execution_time)
+        update_test_file_time(execution_time, path)
+        mark_failed(path) if failed
+      end
+    end
+
     private
 
     def set_defaults
+      @staged_tests = ::Queue.new
       @global_time = 0
       @test_files_with_time = {}
+      @current_batch_failed_paths = []
       @current_test_path = nil
       @prerun_tests_loaded = false
     end
@@ -134,13 +162,13 @@ module KnapsackPro
       @global_time += execution_time
     end
 
-    def update_test_file_time(execution_time)
-      @test_files_with_time[current_test_path] ||= {
+    def update_test_file_time(execution_time, path = current_test_path)
+      @test_files_with_time[path] ||= {
         time_execution: 0,
         measured_time: false,
       }
 
-      hash = @test_files_with_time[current_test_path]
+      hash = @test_files_with_time[path]
 
       if hash[:measured_time]
         hash[:time_execution] += execution_time
@@ -149,7 +177,7 @@ module KnapsackPro
         hash[:measured_time] = true
       end
 
-      @test_files_with_time[current_test_path] = hash
+      @test_files_with_time[path] = hash
     end
 
     def now_without_mock_time

data/lib/knapsack_pro/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module KnapsackPro
-  VERSION = '11.0.0'
+  VERSION = '11.1.0'
 end

data/lib/knapsack_pro.rb

@@ -110,17 +110,17 @@ module KnapsackPro
     end
 
     def logger
-      if KnapsackPro::Config::Env.log_dir
-        default_logger = Logger.new("#{KnapsackPro::Config::Env.log_dir}/knapsack_pro_node_#{KnapsackPro::Config::Env.ci_node_index}.log")
-        default_logger.level = KnapsackPro::Config::Env.log_level
-        self.logger = default_logger
-      end
+      return @logger if @logger
 
-      unless @logger
-        default_logger = ::Logger.new(stdout)
-        default_logger.level = KnapsackPro::Config::Env.log_level
-        self.logger = default_logger
-      end
+      log_dir = KnapsackPro::Config::Env.log_dir
+      default_logger =
+        if log_dir
+          ::Logger.new("#{log_dir}/knapsack_pro_node_#{KnapsackPro::Config::Env.ci_node_index}.log")
+        else
+          ::Logger.new(stdout)
+        end
+      default_logger.level = KnapsackPro::Config::Env.log_level
+      self.logger = default_logger
 
       @logger
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knapsack_pro
 version: !ruby/object:Gem::Version
-  version: 11.0.0
+  version: 11.1.0
 platform: ruby
 authors:
 - ArturT
@@ -57,14 +57,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -242,9 +242,10 @@ files:
 - lib/knapsack_pro/crypto/digestor.rb
 - lib/knapsack_pro/crypto/encryptor.rb
 - lib/knapsack_pro/extensions/rspec_extension.rb
+- lib/knapsack_pro/formatters/cucumber/failures_tracker.rb
+- lib/knapsack_pro/formatters/rspec/time_tracker.rb
+- lib/knapsack_pro/formatters/rspec/time_tracker_fetcher.rb
 - lib/knapsack_pro/formatters/rspec_json_formatter.rb
-- lib/knapsack_pro/formatters/time_tracker.rb
-- lib/knapsack_pro/formatters/time_tracker_fetcher.rb
 - lib/knapsack_pro/hooks/queue.rb
 - lib/knapsack_pro/mask_string.rb
 - lib/knapsack_pro/presenter.rb

data/lib/knapsack_pro/formatters/time_tracker.rb

@@ -1,206 +0,0 @@
-# frozen_string_literal: true
-
-require 'stringio'
-require 'set'
-require_relative '../utils'
-
-module KnapsackPro
-  module Formatters
-    class TimeTracker
-      ::RSpec::Core::Formatters.register self,
-        :example_group_started,
-        :example_started,
-        :example_finished,
-        :example_group_finished
-
-      attr_reader :output # RSpec < v3.10.2
-
-      def initialize(_output)
-        @output = StringIO.new
-        @time_each = nil
-        @time_all = nil
-        @time_all_by_group_id_path = Hash.new(0)
-        @group = {}
-        @paths = {}
-        @suite_started = now
-        @batched_scheduled_paths = []
-        @split_by_test_example_file_paths = Set.new
-        @current_batch_examples = []
-      end
-
-      def current_batch_failed_paths
-        examples_by_file_path = @current_batch_examples
-          .group_by { |example| file_path_for(example) }
-
-        paths =
-          examples_by_file_path.flat_map do |file_path, examples|
-            failed_id_paths = examples.filter { |example| example.execution_result.status.to_s == "failed" }.map(&:id)
-            next [] if failed_id_paths.none?
-            next file_path if KnapsackPro::Config::Env.test_files_encrypted?
-            # Other nodes may have run some examples from this file, it's not safe to compact.
-            next failed_id_paths if rspec_split_by_test_example?(file_path)
-            next file_path if failed_id_paths.size == examples.size
-
-            failed_id_paths
-          end
-
-        paths.map { |path| KnapsackPro::TestFileCleaner.clean(path) }
-      end
-
-      def schedule(paths)
-        @current_batch_examples = []
-        @batched_scheduled_paths << paths
-
-        paths.each do |path|
-          next unless KnapsackPro::Adapters::RSpecAdapter.id_path?(path)
-
-          file_path = KnapsackPro::Adapters::RSpecAdapter.parse_file_path(path)
-          @split_by_test_example_file_paths << KnapsackPro::TestFileCleaner.clean(file_path)
-        end
-      end
-
-      def example_group_started(notification)
-        record_time_all(notification.group.parent_groups[1], @time_all_by_group_id_path, @time_all)
-        @time_all = now
-      end
-
-      def example_started(notification)
-        record_time_all(notification.example.example_group, @time_all_by_group_id_path, @time_all)
-        @time_each = now
-      end
-
-      def example_finished(notification)
-        record_example(@group, notification.example, @time_each)
-        @time_all = now
-      end
-
-      def example_group_finished(notification)
-        record_time_all(notification.group, @time_all_by_group_id_path, @time_all)
-        @time_all = now
-        return unless top_level_group?(notification.group)
-
-        add_hooks_time(@group, @time_all_by_group_id_path)
-        @time_all_by_group_id_path = Hash.new(0)
-        @paths = merge(@paths, @group)
-        @group = {}
-      end
-
-      def queue
-        recorded_paths = @paths.values.map do |example|
-          KnapsackPro::Adapters::RSpecAdapter.parse_file_path(example[:path])
-        end
-
-        missing = (@batched_scheduled_paths.flatten - recorded_paths).each_with_object({}) do |path, object|
-          object[path] = { path: path, time_execution: 0.0 }
-        end
-
-        merge(@paths, missing).values.map do |example|
-          example.transform_keys(&:to_s)
-        end
-      end
-
-      def batch
-        @paths.values.map do |example|
-          example.transform_keys(&:to_s)
-        end
-      end
-
-      def duration
-        now - @suite_started
-      end
-
-      def unexecuted_test_paths
-        pending_paths = @paths.values
-          .filter { |example| example[:time_execution] == 0.0 }
-          .map { |example| example[:path] }
-
-        not_run_paths = @batched_scheduled_paths.flatten -
-          @paths.values
-          .map { |example| example[:path] }
-
-        pending_paths + not_run_paths
-      end
-
-      private
-
-      def top_level_group?(group)
-        group.metadata[:parent_example_group].nil?
-      end
-
-      def add_hooks_time(group, time_all_by_group_id_path)
-        group.each do |_, example|
-          next if example[:time_execution] == 0.0
-
-          example[:time_execution] += time_all_by_group_id_path.reduce(0.0) do |sum, (group_id_path, time)|
-            # :path is a file path (a_spec.rb), sum any before/after(:all) in the file
-            next sum + time if group_id_path.start_with?(example[:path])
-            # :path is an id path (a_spec.rb[1:1]), sum any before/after(:all) above it
-            next sum + time if example[:path].start_with?(group_id_path[0..-2])
-            sum
-          end
-        end
-      end
-
-      def record_example(accumulator, example, started_at)
-        path = path_for(example)
-        return if path.nil?
-
-        @current_batch_examples << example
-
-        time_execution = time_execution_for(example, started_at)
-        if accumulator.key?(path)
-          accumulator[path][:time_execution] += time_execution
-        else
-          accumulator[path] = { path: path, time_execution: time_execution }
-        end
-      end
-
-      def record_time_all(group, time_all_by_group_id_path, time_all)
-        return unless group # above top level group
-
-        group_id_path = KnapsackPro::TestFileCleaner.clean(group.id)
-        time_all_by_group_id_path[group_id_path] += now - time_all
-      end
-
-      def path_for(example)
-        file_path = file_path_for(example)
-        return nil if file_path == ""
-
-        if rspec_split_by_test_example?(file_path)
-          KnapsackPro::TestFileCleaner.clean(example.id)
-        else
-          file_path
-        end
-      end
-
-      def rspec_split_by_test_example?(file_path)
-        @split_by_test_example_file_paths.include?(file_path)
-      end
-
-      def file_path_for(example)
-        KnapsackPro::TestFileCleaner.clean(KnapsackPro::Adapters::RSpecAdapter.file_path_for(example))
-      end
-
-      def time_execution_for(example, started_at)
-        if example.execution_result.status.to_s == "pending"
-          0.0
-        else
-          (now - started_at).to_f
-        end
-      end
-
-      def merge(h1, h2)
-        h1.merge(h2) do |key, v1, v2|
-          {
-            path: key,
-            time_execution: v1[:time_execution] + v2[:time_execution]
-          }
-        end
-      end
-
-      def now
-        KnapsackPro::Utils.time_now
-      end
-    end
-  end
-end

data/lib/knapsack_pro/formatters/time_tracker_fetcher.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module KnapsackPro
-  module Formatters
-    class TimeTrackerFetcher
-      def self.call
-        ::RSpec
-          .configuration
-          .formatters
-          .find { |f| f.class.to_s == "KnapsackPro::Formatters::TimeTracker" }
-      end
-
-      def self.unexecuted_test_paths
-        time_tracker = call
-        return [] unless time_tracker
-        time_tracker.unexecuted_test_paths
-      end
-    end
-  end
-end