knapsack_pro 5.6.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa2ae33f8d962bbd8d5f1639e050f2a43e9ad28bf655d637d43ca2c23bc1d7d3
-  data.tar.gz: 70e747f428235b65bba04ed2f748221170a6971e8982e0ab0fbbdb60177d9a94
+  metadata.gz: 315a4ff2f5885be68529cc0799affd8163fcad51558e4813a254b99bd057992a
+  data.tar.gz: 5709757ca30a22401be4dc170a926b16f5206d494d5b80c583bf03088d8570a6
 SHA512:
-  metadata.gz: ed58c0b63af2f3301e7233c142da4165a9c2c9a4f31987d465a803d3b4fdab590137d2ef71bbf6b80b9abc95f513cea605091b5caa7fcd0598620790977cc29b
-  data.tar.gz: 607af355f302481ca595f577428ea49565e711a790c284d7fb595f0f15106070a99e85acf4ebd5cf0e073e672ee37f9d851eb2b464c372dca550bf5c878c8050
+  metadata.gz: c43ee2474109d5e509fb57534857164ec0596cb018f61b44942ab261bdcb55345ac3025239fd3dde12c7d31f30841aa39aa480fb07bcab1519272788dad9c4b7
+  data.tar.gz: 9511b45fb2b0d59a309517ccc70ca10ebbb2b8a3cf0cfb2f2198e38a776838a402b531f64aea2ca94ce37489288dde8171b5fd36c45038014d781cba0fd5462d

data/.circleci/config.yml

@@ -41,3 +41,5 @@ jobs:
       - run: rubocop -A --only Style/FrozenStringLiteralComment,Layout/EmptyLineAfterMagicComment lib/
 
       - run: bundle exec rspec spec
+
+      - run: bundle exec ruby spec/knapsack_pro/formatters/time_tracker_specs.rb

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+### 6.0.0
+
+* __(breaking change)__ Dropped support for Turnip < 2.0.0
+* Use an RSpec Formatter to track tests' execution times more accurately
+* Removed `Time.raw_now`
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/229
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v5.7.0...UNRELEASED
+
+### 5.7.0
+
+* Performance improvement: don't run `rake knapsack_pro:rspec_test_example_detector` when no slow test files are detected for RSpec.
+
+    https://github.com/KnapsackPro/knapsack_pro-ruby/pull/225
+
+https://github.com/KnapsackPro/knapsack_pro-ruby/compare/v5.6.0...v5.7.0
+
 ### 5.6.0
 
 * Use `frozen_string_literal: true` to reduce memory usage

data/README.md

@@ -74,7 +74,7 @@ bundle update knapsack_pro
 RSpec:
 
 ```bash
-bundle exec rspec spec
+bin/test
 ```
 
 Scripted tests can be found in the [Rails App With Knapsack Pro repository](https://github.com/KnapsackPro/rails-app-with-knapsack_pro/blob/master/bin/knapsack_pro_all.rb).

data/bin/test

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+bundle exec ruby spec/knapsack_pro/formatters/time_tracker_specs.rb
+export FORMATTERS_EXIT_CODE=$?
+
+bundle exec rspec spec
+export RSPEC_EXIT_CODE=$?
+
+if [ "$FORMATTERS_EXIT_CODE" -ne "0" ]; then
+  exit $FORMATTERS_EXIT_CODE
+fi
+
+if [ "$RSPEC_EXIT_CODE" -ne "0" ]; then
+  exit $RSPEC_EXIT_CODE
+fi

data/knapsack_pro.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'knapsack_pro/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "knapsack_pro"
+  spec.name          = 'knapsack_pro'
   spec.version       = KnapsackPro::VERSION
   spec.authors       = ['ArturT']
-  spec.email         = ['arturtrzop@gmail.com']
+  spec.email         = ['support@knapsackpro.com']
   spec.summary       = %q{Knapsack Pro splits tests across parallel CI nodes and ensures each parallel job finish work at a similar time.}
   spec.description   = %q{Run tests in parallel across CI server nodes based on tests execution time. Split tests in a dynamic way to ensure parallel jobs are done at a similar time. Thanks to that your CI build time is as fast as possible. It works with many CI providers.}
   spec.homepage      = 'https://knapsackpro.com'
@@ -15,15 +15,15 @@ Gem::Specification.new do |spec|
   spec.metadata    = {
     'bug_tracker_uri' => 'https://github.com/KnapsackPro/knapsack_pro-ruby/issues',
     'changelog_uri' => 'https://github.com/KnapsackPro/knapsack_pro-ruby/blob/master/CHANGELOG.md',
-    'documentation_uri' => 'https://docs.knapsackpro.com/integration/',
+    'documentation_uri' => 'https://docs.knapsackpro.com/knapsack_pro-ruby/guide/',
     'homepage_uri' => 'https://knapsackpro.com',
     'source_code_uri' => 'https://github.com/KnapsackPro/knapsack_pro-ruby'
   }
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = ['knapsack_pro']
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.add_dependency 'rake', '>= 0'
 

data/lib/knapsack_pro/adapters/base_adapter.rb

@@ -11,6 +11,14 @@ module KnapsackPro
         "#{KnapsackPro::Config::TempFiles::TEMP_DIRECTORY_PATH}/#{adapter_name}-bind_method_called_for_node_#{KnapsackPro::Config::Env.ci_node_index}.txt"
       end
 
+      def self.split_by_test_cases_enabled?
+        false
+      end
+
+      def self.test_file_cases_for(slow_test_files)
+        raise NotImplementedError
+      end
+
       def self.slow_test_file?(adapter_class, test_file_path)
         @slow_test_file_paths ||=
           begin

data/lib/knapsack_pro/adapters/rspec_adapter.rb

@@ -1,10 +1,41 @@
 # frozen_string_literal: true
 
+require_relative '../formatters/time_tracker_fetcher'
+
 module KnapsackPro
   module Adapters
     class RSpecAdapter < BaseAdapter
       TEST_DIR_PATTERN = 'spec/**{,/*/**}/*_spec.rb'
 
+      def self.split_by_test_cases_enabled?
+        return false unless KnapsackPro::Config::Env.rspec_split_by_test_examples?
+
+        require 'rspec/core/version'
+        unless Gem::Version.new(::RSpec::Core::Version::STRING) >= Gem::Version.new('3.3.0')
+          raise "RSpec >= 3.3.0 is required to split test files by test examples. Learn more: #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}"
+        end
+
+        true
+      end
+
+      def self.test_file_cases_for(slow_test_files)
+        KnapsackPro.logger.info("Generating RSpec test examples JSON report for slow test files to prepare it to be split by test examples (by individual test cases). Thanks to that, a single slow test file can be split across parallel CI nodes. Analyzing #{slow_test_files.size} slow test files.")
+
+        # generate the RSpec JSON report in a separate process to not pollute the RSpec state
+        cmd = [
+          'RACK_ENV=test',
+          'RAILS_ENV=test',
+          KnapsackPro::Config::Env.rspec_test_example_detector_prefix,
+          'rake knapsack_pro:rspec_test_example_detector',
+        ].join(' ')
+        unless Kernel.system(cmd)
+          raise "Could not generate JSON report for RSpec. Rake task failed when running #{cmd}"
+        end
+
+        # read the JSON report
+        KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new.test_file_example_paths
+      end
+
       def self.ensure_no_tag_option_when_rspec_split_by_test_examples_enabled!(cli_args)
         if KnapsackPro::Config::Env.rspec_split_by_test_examples? && has_tag_option?(cli_args)
           error_message = "It is not allowed to use the RSpec tag option together with the RSpec split by test examples feature. Please see: #{KnapsackPro::Urls::RSPEC__SPLIT_BY_TEST_EXAMPLES__TAG}"
@@ -25,63 +56,61 @@ module KnapsackPro
         parsed_options(cli_args)&.[](:order)
       end
 
-      def self.test_path(example)
-        example_group = example.metadata[:example_group]
-
-        if defined?(::Turnip) && Gem::Version.new(::Turnip::VERSION) < Gem::Version.new('2.0.0')
-          unless example_group[:turnip]
-            until example_group[:parent_example_group].nil?
-              example_group = example_group[:parent_example_group]
-            end
+      def self.file_path_for(example)
+        [
+          -> { parse_file_path(example.id) },
+          -> { example.metadata[:file_path] },
+          -> { example.metadata[:example_group][:file_path] },
+          -> { top_level_group(example)[:file_path] },
+        ]
+          .each do |path|
+            p = path.call
+            return p if p.include?('_spec.rb')
           end
-        else
-          until example_group[:parent_example_group].nil?
-            example_group = example_group[:parent_example_group]
-          end
-        end
 
-        example_group[:file_path]
+        return ''
       end
 
-      def bind_time_tracker
-        ::RSpec.configure do |config|
-          config.prepend_before(:context) do
-            KnapsackPro.tracker.start_timer
-          end
+      def self.parse_file_path(id)
+        # https://github.com/rspec/rspec-core/blob/1eeadce5aa7137ead054783c31ff35cbfe9d07cc/lib/rspec/core/example.rb#L122
+        id.match(/\A(.*?)(?:\[([\d\s:,]+)\])?\z/).captures.first
+      end
 
-          config.around(:each) do |example|
-            current_test_path = KnapsackPro::Adapters::RSpecAdapter.test_path(example)
-
-            # Stop timer to update time for a previously run test example.
-            # This way we count time spent in runtime for the previous test example after around(:each) is already done.
-            # Only do that if we're in the same test file. Otherwise, `before(:all)` execution time in the current file
-            # will be applied to the previously ran test file.
-            if KnapsackPro.tracker.current_test_path&.start_with?(KnapsackPro::TestFileCleaner.clean(current_test_path))
-              KnapsackPro.tracker.stop_timer
-            end
+      # private
+      def self.top_level_group(example)
+        group = example.metadata[:example_group]
+        until group[:parent_example_group].nil?
+          group = group[:parent_example_group]
+        end
+        group
+      end
 
-            KnapsackPro.tracker.current_test_path =
-              if KnapsackPro::Config::Env.rspec_split_by_test_examples? && KnapsackPro::Adapters::RSpecAdapter.slow_test_file?(RSpecAdapter, current_test_path)
-                example.id
-              else
-                current_test_path
-              end
+      def bind_time_tracker
+        ensure_no_focus!
+        log_batch_duration
+      end
 
+      def ensure_no_focus!
+        ::RSpec.configure do |config|
+          config.around(:each) do |example|
             if example.metadata[:focus] && KnapsackPro::Adapters::RSpecAdapter.rspec_configuration.filter.rules[:focus]
-              raise "We detected a test file path #{current_test_path} with a test using the metadata `:focus` tag. RSpec might not run some tests in the Queue Mode (causing random tests skipping problem). Please remove the `:focus` tag from your codebase. See more: #{KnapsackPro::Urls::RSPEC__SKIPS_TESTS}"
+              file_path = KnapsackPro::Adapters::RSpecAdapter.file_path_for(example)
+              file_path = KnapsackPro::TestFileCleaner.clean(file_path)
+
+              raise "Knapsack Pro found an example tagged with focus in #{file_path}, please remove it. See more: #{KnapsackPro::Urls::RSPEC__SKIPS_TESTS}"
             end
 
             example.run
           end
+        end
+      end
 
-          config.append_after(:context) do
-            # after(:context) hook is run one time only, after all of the examples in a group
-            # stop timer to count time for the very last executed test example
-            KnapsackPro.tracker.stop_timer
-          end
-
+      def log_batch_duration
+        ::RSpec.configure do |config|
           config.after(:suite) do
-            KnapsackPro.logger.debug(KnapsackPro::Presenter.global_time)
+            time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+            formatted = KnapsackPro::Presenter.global_time(time_tracker.batch_duration)
+            KnapsackPro.logger.debug(formatted)
           end
         end
       end
@@ -89,7 +118,8 @@ module KnapsackPro
       def bind_save_report
         ::RSpec.configure do |config|
           config.after(:suite) do
-            KnapsackPro::Report.save
+            time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+            KnapsackPro::Report.save(time_tracker.batch)
           end
         end
       end

data/lib/knapsack_pro/base_allocator_builder.rb

@@ -35,35 +35,16 @@ module KnapsackPro
     def fast_and_slow_test_files_to_run
       test_files_to_run = all_test_files_to_run
 
-      if adapter_class == KnapsackPro::Adapters::RSpecAdapter && KnapsackPro::Config::Env.rspec_split_by_test_examples?
-        require 'rspec/core/version'
-        unless Gem::Version.new(::RSpec::Core::Version::STRING) >= Gem::Version.new('3.3.0')
-          raise "RSpec >= 3.3.0 is required to split test files by test examples. Learn more: #{KnapsackPro::Urls::SPLIT_BY_TEST_EXAMPLES}"
-        end
-
+      if adapter_class.split_by_test_cases_enabled?
         slow_test_files = get_slow_test_files
+        return test_files_to_run if slow_test_files.empty?
 
-        KnapsackPro.logger.info("Generating RSpec test examples JSON report for slow test files to prepare it to be split by test examples (by individual 'it's. Thanks to that a single slow test file can be split across parallel CI nodes). Analyzing #{slow_test_files.size} slow test files.")
-
-        # generate RSpec JSON report in separate process to not pollute RSpec state
-        cmd = [
-          'RACK_ENV=test',
-          'RAILS_ENV=test',
-          KnapsackPro::Config::Env.rspec_test_example_detector_prefix,
-          'rake knapsack_pro:rspec_test_example_detector',
-        ].join(' ')
-        unless Kernel.system(cmd)
-          raise "Could not generate JSON report for RSpec. Rake task failed when running #{cmd}"
-        end
+        test_file_cases = adapter_class.test_file_cases_for(slow_test_files)
 
-        # read JSON report
-        detector = KnapsackPro::TestCaseDetectors::RSpecTestExampleDetector.new
-        test_file_example_paths = detector.test_file_example_paths
-
-        KnapsackPro::TestFilesWithTestCasesComposer.call(test_files_to_run, slow_test_files, test_file_example_paths)
-      else
-        test_files_to_run
+        return KnapsackPro::TestFilesWithTestCasesComposer.call(test_files_to_run, slow_test_files, test_file_cases)
       end
+
+      test_files_to_run
     end
 
     private

data/lib/knapsack_pro/formatters/rspec_queue_summary_formatter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative './time_tracker_fetcher'
+
 RSpec::Support.require_rspec_core('formatters/base_formatter')
 RSpec::Support.require_rspec_core('formatters/base_text_formatter')
 
@@ -76,11 +78,12 @@ module KnapsackPro
         registered_output.puts(most_recent_summary)
       end
 
-      def self.print_exit_summary
+      def self.print_exit_summary(all_test_file_paths)
         registered_output.puts('Knapsack Pro Queue exited/aborted!')
         registered_output.puts('')
 
-        unexecuted_test_files = KnapsackPro.tracker.unexecuted_test_files
+        time_tracker = KnapsackPro::Formatters::TimeTrackerFetcher.call
+        unexecuted_test_files = time_tracker&.unexecuted_test_files(all_test_file_paths) || []
         unless unexecuted_test_files.empty?
           registered_output.puts('Unexecuted tests on this CI node:')
           registered_output.puts(unexecuted_test_files)
@@ -119,7 +122,7 @@ module KnapsackPro
 
       def dump_summary(summary)
         colorizer = ::RSpec::Core::Formatters::ConsoleCodes
-        duration = KnapsackPro.tracker.global_time_since_beginning
+        duration = KnapsackPro::Formatters::TimeTrackerFetcher.call.duration
         formatted_duration = ::RSpec::Core::Formatters::Helpers.format_duration(duration)
 
         formatted = "\nFinished in #{formatted_duration}\n" \

data/lib/knapsack_pro/formatters/time_tracker.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module KnapsackPro
+  module Formatters
+    class TimeTracker
+      ::RSpec::Core::Formatters.register self,
+        :example_group_started,
+        :example_started,
+        :example_finished,
+        :example_group_finished,
+        :stop
+
+      # Called at the beginning of each batch,
+      # but only the first instance of this class is used,
+      # so don't rely on the initializer to reset values.
+      def initialize(_output)
+        @time_each = nil
+        @time_all = nil
+        @before_all = 0.0
+        @group = {}
+        @batch = {}
+        @queue = {}
+        @suite_started = now
+        @batch_started = now
+      end
+
+      def example_group_started(notification)
+        return unless top_level_group?(notification.group)
+        @time_all = now
+      end
+
+      def example_started(_notification)
+        @before_all = now - @time_all if @before_all == 0.0
+        @time_each = now
+      end
+
+      def example_finished(notification)
+        record_example(@group, notification.example, @time_each)
+        @time_all = now
+      end
+
+      def example_group_finished(notification)
+        return unless top_level_group?(notification.group)
+
+        add_hooks_time(@group, @before_all, now - @time_all)
+        @batch = merge(@batch, @group)
+        @before_all = 0.0
+        @group = {}
+      end
+
+      # Called at the end of each batch
+      def stop(_notification)
+        @queue = merge(@queue, @batch)
+        @batch = {}
+        @batch_started = now
+      end
+
+      def queue(scheduled_paths)
+        recorded_paths = @queue.values.map do |example|
+          KnapsackPro::Adapters::RSpecAdapter.parse_file_path(example[:path])
+        end
+
+        missing = (scheduled_paths - recorded_paths).each_with_object({}) do |path, object|
+          object[path] = { path: path, time_execution: 0.0 }
+        end
+
+        merge(@queue, missing).values.map do |example|
+          example.transform_keys(&:to_s)
+        end
+      end
+
+      def batch
+        @batch.values.map do |example|
+          example.transform_keys(&:to_s)
+        end
+      end
+
+      def duration
+        now - @suite_started
+      end
+
+      def batch_duration
+        now - @batch_started
+      end
+
+      def unexecuted_test_files(scheduled_paths)
+        pending_paths = (@queue.values + @batch.values)
+          .filter { |example| example[:time_execution] == 0.0 }
+          .map { |example| example[:path] }
+
+        not_run_paths = scheduled_paths -
+          (@queue.values + @batch.values)
+          .map { |example| example[:path] }
+
+        pending_paths + not_run_paths
+      end
+
+      private
+
+      def top_level_group?(group)
+        group.metadata[:parent_example_group].nil?
+      end
+
+      def add_hooks_time(group, before_all, after_all)
+        group.each do |_, example|
+          next if example[:time_execution] == 0.0
+          example[:time_execution] += before_all + after_all
+        end
+      end
+
+      def record_example(accumulator, example, started_at)
+        path = path_for(example)
+        time_execution = time_execution_for(example, started_at)
+
+        if accumulator.key?(path)
+          accumulator[path][:time_execution] += time_execution
+        else
+          accumulator[path] = { path: path, time_execution: time_execution }
+        end
+      end
+
+      def path_for(example)
+        file = file_path_for(example)
+        return "UNKNOWN_PATH" if file == ""
+        path = rspec_split_by_test_example?(file) ? example.id : file
+        KnapsackPro::TestFileCleaner.clean(path)
+      end
+
+      def rspec_split_by_test_example?(file)
+        return false unless KnapsackPro::Config::Env.rspec_split_by_test_examples?
+        return false unless KnapsackPro::Adapters::RSpecAdapter.slow_test_file?(KnapsackPro::Adapters::RSpecAdapter, file)
+        true
+      end
+
+      def file_path_for(example)
+        KnapsackPro::Adapters::RSpecAdapter.file_path_for(example)
+      end
+
+      def time_execution_for(example, started_at)
+        if example.execution_result.status.to_s == "pending"
+          0.0
+        else
+          (now - started_at).to_f
+        end
+      end
+
+      def merge(h1, h2)
+        h1.merge(h2) do |key, v1, v2|
+          {
+            path: key,
+            time_execution: v1[:time_execution] + v2[:time_execution]
+          }
+        end
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/knapsack_pro/formatters/time_tracker_fetcher.rb

@@ -0,0 +1,12 @@
+module KnapsackPro
+  module Formatters
+    class TimeTrackerFetcher
+      def self.call
+        ::RSpec
+          .configuration
+          .formatters
+          .find { |f| f.class.to_s == "KnapsackPro::Formatters::TimeTracker" }
+      end
+    end
+  end
+end

data/lib/knapsack_pro/presenter.rb

@@ -3,8 +3,9 @@
 module KnapsackPro
   class Presenter
     class << self
-      def global_time
-        global_time = pretty_seconds(KnapsackPro.tracker.global_time)
+      def global_time(time = nil)
+        time = KnapsackPro.tracker.global_time if time.nil?
+        global_time = pretty_seconds(time)
         "Global time execution for tests: #{global_time}"
       end
 

data/lib/knapsack_pro/report.rb

@@ -2,15 +2,17 @@
 
 module KnapsackPro
   class Report
-    def self.save
-      test_files = KnapsackPro.tracker.to_a
+    def self.save(tests = nil)
+      if tests.nil?
+        tests = KnapsackPro.tracker.to_a
+      end
 
-      if test_files.empty?
+      if tests.empty?
         KnapsackPro.logger.warn("No test files were executed on this CI node.")
         KnapsackPro.logger.debug("When you use knapsack_pro Regular Mode, the reason for no tests executing might be a very narrow tests list. Most likely, you run only tests with a specified tag, and there were fewer test files with the tag than parallel CI nodes.")
       end
 
-      create_build_subset(test_files)
+      create_build_subset(tests)
     end
 
     def self.save_subset_queue_to_file
@@ -30,11 +32,13 @@ module KnapsackPro
       end
     end
 
-    def self.save_node_queue_to_api
-      test_files = []
-      Dir.glob("#{queue_path}/*.json").each do |file|
-        report = JSON.parse(File.read(file))
-        test_files += report
+    def self.save_node_queue_to_api(test_files = nil)
+      if test_files.nil?
+        test_files = []
+        Dir.glob("#{queue_path}/*.json").each do |file|
+          report = JSON.parse(File.read(file))
+          test_files += report
+        end
       end
 
       if test_files.empty?