test-prof 1.3.0 → 1.4.4

data/lib/test_prof/memory_prof/tracker.rb

@@ -36,22 +36,26 @@ module TestProf
         @total_memory = node.total_memory
       end
 
-      def example_started(example)
-        list.add_node(example, track)
+      def example_started(id, example = id)
+        list.add_node(id, example, track)
       end
 
-      def example_finished(example)
-        node = list.remove_node(example, track)
-        examples << {**example, memory: node.total_memory}
+      def example_finished(id)
+        node = list.remove_node(id, track)
+        return unless node
+
+        examples << {**node.item, memory: node.total_memory}
       end
 
-      def group_started(group)
-        list.add_node(group, track)
+      def group_started(id, group = id)
+        list.add_node(id, group, track)
       end
 
-      def group_finished(group)
-        node = list.remove_node(group, track)
-        groups << {**group, memory: node.hooks_memory}
+      def group_finished(id)
+        node = list.remove_node(id, track)
+        return unless node
+
+        groups << {**node.item, memory: node.hooks_memory}
       end
     end
 

data/lib/test_prof/recipes/minitest/before_all.rb

@@ -104,7 +104,7 @@ module TestProf
           if base.respond_to?(:parallelize_teardown)
             base.parallelize_teardown do
               last_klass = ::Minitest.previous_klass
-              if last_klass&.respond_to?(:parallelized) && last_klass&.parallelized
+              if last_klass&.respond_to?(:parallelized) && last_klass.parallelized
                 last_klass.before_all_executor&.deactivate!
               end
             end
@@ -114,7 +114,7 @@ module TestProf
             base.singleton_class.prepend(Module.new do
               def parallelize(workers: :number_of_processors, with: :processes)
                 # super.parallelize returns nil when no parallelization is set up
-                if super(workers: workers, with: with).nil?
+                if super.nil?
                   return
                 end
 

data/lib/test_prof/recipes/minitest/sample.rb

@@ -7,14 +7,16 @@ module TestProf
     CORE_RUNNABLES = [
       Minitest::Test,
       defined?(Minitest::Unit::TestCase) ? Minitest::Unit::TestCase : nil,
-      Minitest::Spec
+      defined?(Minitest::Spec) ? Minitest::Spec : nil
     ].compact.freeze
 
     class << self
       def suites
         # Make sure that sample contains only _real_ suites
         Minitest::Runnable.runnables
-          .reject { |suite| CORE_RUNNABLES.include?(suite) }
+          .select do |suite|
+            CORE_RUNNABLES.any? { |kl| suite < kl } && suite.runnable_methods.any?
+          end
       end
 
       def sample_groups(sample_size)
@@ -27,11 +29,18 @@ module TestProf
         all_examples = suites.flat_map do |runnable|
           runnable.runnable_methods.map { |method| [runnable, method] }
         end
-        sample = all_examples.sample(sample_size)
+
+        sample = all_examples.sample(sample_size).group_by(&:first)
+        sample.transform_values! { |v| v.map(&:last) }
+
         # Filter examples by overriding #runnable_methods for all suites
         suites.each do |runnable|
-          runnable.define_singleton_method(:runnable_methods) do
-            super() & sample.select { |ex| ex.first.equal?(runnable) }.map(&:last)
+          if sample.key?(runnable)
+            runnable.define_singleton_method(:runnable_methods) do
+              super() & sample[runnable]
+            end
+          else
+            runnable.define_singleton_method(:runnable_methods) { [] }
           end
         end
       end

data/lib/test_prof/recipes/rspec/any_fixture.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "test_prof/any_fixture"
+require "test_prof/any_fixture/dsl"
+require "test_prof/ext/active_record_refind"
+
 require "test_prof/recipes/rspec/before_all"
 
 RSpec.shared_context "any_fixture:clean" do
@@ -8,6 +11,11 @@ RSpec.shared_context "any_fixture:clean" do
 
   before_all do
     TestProf::AnyFixture.clean
+    TestProf::AnyFixture.disable!
+  end
+
+  after(:all) do
+    TestProf::AnyFixture.enable!
   end
 end
 

data/lib/test_prof/recipes/rspec/let_it_be.rb

@@ -7,6 +7,8 @@ module TestProf
   # Just like `let`, but persist the result for the whole group.
   # NOTE: Experimental and magical, for more control use `before_all`.
   module LetItBe
+    class DuplicationError < StandardError; end
+
     Modifier = Struct.new(:scope, :block) do
       def call(record, config)
         block.call(record, config)
@@ -14,6 +16,8 @@ module TestProf
     end
 
     class Configuration
+      attr_accessor :report_duplicates
+
       # Define an alias for `let_it_be` with the predefined options:
       #
       #   TestProf::LetItBe.configure do |config|
@@ -117,6 +121,8 @@ module TestProf
         instance_variable_get(:"#{PREFIX}#{identifier}")
       end
 
+      report_duplicates(identifier) if LetItBe.config.report_duplicates
+
       LetItBe.module_for(self).module_eval do
         define_method(identifier) do
           # Trying to detect the context
@@ -135,6 +141,19 @@ module TestProf
       let(identifier, &let_accessor)
     end
 
+    private def report_duplicates(identifier)
+      if instance_methods.include?(identifier) && File.basename(__FILE__) == File.basename(instance_method(identifier).source_location[0])
+        error_msg = "let_it_be(:#{identifier}) was redefined in nested group"
+        report_level = LetItBe.config.report_duplicates.to_sym
+
+        if report_level == :warn
+          ::RSpec.warn_with(error_msg)
+        elsif report_level == :raise
+          raise DuplicationError, error_msg
+        end
+      end
+    end
+
     module Freezer
       # Stoplist to prevent freezing objects and theirs associations that are defined
       # with `let_it_be`'s `freeze: false` options during deep freezing.

data/lib/test_prof/rspec_stamp/parser.rb

@@ -143,7 +143,7 @@ module TestProf
           elsif expr.first == :@const
             expr[1]
           elsif expr.first == :const_path_ref
-            expr[1..-1].map(&method(:parse_const)).join("::")
+            expr[1..].map(&method(:parse_const)).join("::")
           end
         end
       end

data/lib/test_prof/rspec_stamp.rb

@@ -153,14 +153,16 @@ module TestProf
           end
         end
 
-        replacement = "\\1#{parsed.fname}#{need_parens ? "(" : " "}" \
-                      "#{[desc, tags_str, htags_str].compact.join(", ")}" \
-                      "#{need_parens ? ") " : " "}\\3"
+        replacement = proc do
+          $1 + "#{parsed.fname}#{need_parens ? "(" : " "}" \
+          "#{[desc, tags_str, htags_str].compact.join(", ")}" \
+          "#{need_parens ? ") " : " "}" + $3
+        end
 
         if config.dry_run?
-          log :info, "Patched: #{example.sub(EXAMPLE_RXP, replacement)}"
+          log :info, "Patched: #{example.sub(EXAMPLE_RXP, &replacement)}"
         else
-          example.sub!(EXAMPLE_RXP, replacement)
+          example.sub!(EXAMPLE_RXP, &replacement)
         end
         true
       end

data/lib/test_prof/ruby_prof/rspec_no_boot.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  report = nil
+
+  config.append_before(:suite) do
+    report = TestProf::RubyProf.profile(locked: true)
+
+    TestProf.log :info, "RubyProf enabled for examples"
+  end
+
+  config.after(:suite) do
+    report&.dump("examples")
+  end
+end

data/lib/test_prof/ruby_prof.rb

@@ -48,12 +48,13 @@ module TestProf
       attr_accessor :printer, :mode, :min_percent,
         :include_threads, :exclude_common_methods,
         :test_prof_exclusions_enabled,
-        :custom_exclusions
+        :custom_exclusions, :skip_boot
 
       def initialize
         @printer = ENV["TEST_RUBY_PROF"].to_sym if PRINTERS.key?(ENV["TEST_RUBY_PROF"])
         @printer ||= ENV.fetch("TEST_RUBY_PROF_PRINTER", :flat).to_sym
         @mode = ENV.fetch("TEST_RUBY_PROF_MODE", :wall).to_s
+        @skip_boot = %w[0 false f].include?(ENV["TEST_RUBY_PROF_BOOT"])
         @min_percent = 1
         @include_threads = false
         @exclude_common_methods = true
@@ -65,6 +66,10 @@ module TestProf
         include_threads == true
       end
 
+      def skip_boot?
+        skip_boot == true
+      end
+
       def exclude_common_methods?
         exclude_common_methods == true
       end
@@ -166,23 +171,21 @@ module TestProf
       #
       # Use this method to profile the whole run.
       def run
-        report = profile
+        report = profile(locked: true)
 
         return unless report
 
-        @locked = true
-
         log :info, "RubyProf enabled globally"
 
         at_exit { report.dump("total") }
       end
 
-      def profile
+      def profile(locked: false)
         if locked?
           log :warn, <<~MSG
             RubyProf is activated globally, you cannot generate per-example report.
 
-            Make sure you haven't set the TEST_RUBY_PROF environmental variable.
+            Make sure you haven not set the TEST_RUBY_PROF environmental variable.
           MSG
           return
         end
@@ -212,6 +215,8 @@ module TestProf
 
         profiler.start
 
+        @locked = true if locked
+
         Report.new(profiler)
       end
 
@@ -282,5 +287,13 @@ end
 
 # Hook to run RubyProf globally
 TestProf.activate("TEST_RUBY_PROF") do
-  TestProf::RubyProf.run
+  if TestProf::RubyProf.config.skip_boot?
+    if TestProf.rspec?
+      require "test_prof/ruby_prof/rspec_no_boot"
+    else
+      TestProf.log :warn, "RubyProf tests profiling w/o test suite boot is only supported in RSpec"
+    end
+  else
+    TestProf::RubyProf.run
+  end
 end

data/lib/test_prof/tag_prof/minitest.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Minitest
+  module TestProf
+    class TagProfReporter < BaseReporter # :nodoc:
+      attr_reader :results
+
+      def initialize(io = $stdout, _options = {})
+        super
+        @results = ::TestProf::TagProf::Result.new("type")
+
+        if event_prof_activated?
+          require "test_prof/event_prof"
+          @current_group_id = nil
+          @events_profiler = configure_profiler
+          @results = ::TestProf::TagProf::Result.new("type", @events_profiler.events)
+        end
+      end
+
+      def prerecord(group, example)
+        return unless event_prof_activated?
+
+        # enable event profiling
+        @events_profiler.group_started(true)
+      end
+
+      def record(result)
+        results.track(main_folder_path(result), time: result.time, events: fetch_events_data)
+        @events_profiler.group_started(nil) if event_prof_activated? # reset and disable event profilers
+      end
+
+      def report
+        printer = (ENV["TAG_PROF_FORMAT"] == "html") ? ::TestProf::TagProf::Printers::HTML : ::TestProf::TagProf::Printers::Simple
+        printer.dump(results)
+      end
+
+      private
+
+      def main_folder_path(result)
+        return :__unknown__ if absolute_path_from(result).nil?
+
+        absolute_path_from(result)
+      end
+
+      def absolute_path_from(result)
+        absolute_path = File.expand_path(result.source_location.first)
+        absolute_path.slice(/(?<=(?:spec|test)\/)\w*/)
+      end
+
+      def configure_profiler
+        ::TestProf::EventProf::CustomEvents.activate_all(tag_prof_event)
+        ::TestProf::EventProf.build(tag_prof_event)
+      end
+
+      def event_prof_activated?
+        return false if tag_prof_event.nil?
+
+        !tag_prof_event.empty?
+      end
+
+      def tag_prof_event
+        ENV["TAG_PROF_EVENT"]
+      end
+
+      def fetch_events_data
+        return {} unless @events_profiler
+
+        @events_profiler.profilers.map do |profiler|
+          [profiler.event, profiler.time || 0.0]
+        end.to_h
+      end
+    end
+  end
+end

data/lib/test_prof/tag_prof/result.rb

@@ -21,8 +21,8 @@ module TestProf
       def track(tag, time:, events: {})
         data[tag][:count] += 1
         data[tag][:time] += time
-        events.each do |k, v|
-          data[tag][k] += v
+        events.each do |event, time|
+          data[tag][event] += time
         end
       end
 

data/lib/test_prof/tps_prof/profiler.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "test_prof/utils/sized_ordered_set"
+
+module TestProf
+  module TPSProf
+    class Profiler
+      attr_reader :top_count, :groups, :total_count, :total_time, :threshold
+
+      def initialize(top_count, threshold: 10)
+        @threshold = threshold
+        @top_count = top_count
+        @total_count = 0
+        @total_time = 0.0
+        @groups = Utils::SizedOrderedSet.new(top_count, sort_by: :tps)
+      end
+
+      def group_started(id)
+        @current_group = id
+        @examples_count = 0
+        @examples_time = 0.0
+        @group_started_at = TestProf.now
+      end
+
+      def group_finished(id)
+        return unless @examples_count >= threshold
+
+        # Context-time
+        group_time = (TestProf.now - @group_started_at) - @examples_time
+        run_time = @examples_time + group_time
+
+        groups << {
+          id: id,
+          run_time: run_time,
+          group_time: group_time,
+          count: @examples_count,
+          tps: -(@examples_count / run_time).round(2)
+        }
+      end
+
+      def example_started(id)
+        @example_started_at = TestProf.now
+      end
+
+      def example_finished(id)
+        @examples_count += 1
+        @total_count += 1
+
+        time = (TestProf.now - @example_started_at)
+        @examples_time += time
+        @total_time += time
+      end
+    end
+  end
+end

data/lib/test_prof/tps_prof/reporter/text.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "test_prof/ext/float_duration"
+require "test_prof/ext/string_truncate"
+
+module TestProf
+  module TPSProf
+    module Reporter
+      class Text
+        include Logging
+        using FloatDuration
+        using StringTruncate
+
+        def print(profiler)
+          groups = profiler.groups
+
+          total_tps = (profiler.total_count / profiler.total_time).round(2)
+
+          msgs = []
+
+          msgs <<
+            <<~MSG
+              Total TPS (tests per second): #{total_tps}
+
+              Top #{profiler.top_count} slowest suites by TPS (tests per second) (min examples per group: #{profiler.threshold}):
+
+            MSG
+
+          groups.each do |group|
+            description = group[:id].top_level_description
+            location = group[:id].metadata[:location]
+            time = group[:run_time]
+            group_time = group[:group_time]
+            count = group[:count]
+            tps = -group[:tps]
+
+            msgs <<
+              <<~GROUP
+                #{description.truncate} (#{location}) – #{tps} TPS (#{time.duration} / #{count}), group time: #{group_time.duration}
+              GROUP
+          end
+
+          log :info, msgs.join
+        end
+      end
+    end
+  end
+end

data/lib/test_prof/tps_prof/rspec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module TestProf
+  module TPSProf
+    class RSpecListener # :nodoc:
+      include Logging
+
+      NOTIFICATIONS = %i[
+        example_group_started
+        example_group_finished
+        example_started
+        example_finished
+      ].freeze
+
+      attr_reader :reporter, :profiler
+
+      def initialize
+        @profiler = Profiler.new(TPSProf.config.top_count, threshold: TPSProf.config.threshold)
+        @reporter = TPSProf.config.reporter
+
+        log :info, "TPSProf enabled (top-#{TPSProf.config.top_count})"
+      end
+
+      def example_group_started(notification)
+        return unless notification.group.top_level?
+        profiler.group_started notification.group
+      end
+
+      def example_group_finished(notification)
+        return unless notification.group.top_level?
+        profiler.group_finished notification.group
+      end
+
+      def example_started(notification)
+        profiler.example_started notification.example
+      end
+
+      def example_finished(notification)
+        profiler.example_finished notification.example
+      end
+
+      def print
+        reporter.print(profiler)
+      end
+    end
+  end
+end
+
+# Register TPSProf listener
+TestProf.activate("TPS_PROF") do
+  RSpec.configure do |config|
+    listener = nil
+
+    config.before(:suite) do
+      listener = TestProf::TPSProf::RSpecListener.new
+      config.reporter.register_listener(
+        listener, *TestProf::TPSProf::RSpecListener::NOTIFICATIONS
+      )
+    end
+
+    config.after(:suite) { listener&.print }
+  end
+end

data/lib/test_prof/tps_prof.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "test_prof/tps_prof/profiler"
+require "test_prof/tps_prof/reporter/text"
+
+module TestProf
+  # TPSProf shows top-N example group based on their tests-per-second value.
+  #
+  # Example:
+  #
+  #   TPS_PROF=10 rspec ...
+  #
+  module TPSProf
+    class Configuration
+      attr_accessor :top_count, :threshold, :reporter
+
+      def initialize
+        @top_count = ENV["TPS_PROF"].to_i
+        @top_count = 10 if @top_count == 1
+        @threshold = ENV.fetch("TPS_PROF_MIN", 10).to_i
+        @reporter = resolve_reporter(ENV["TPS_PROF_FORMAT"])
+      end
+
+      private
+
+      def resolve_reporter(format)
+        # TODO: support other formats
+        TPSProf::Reporter::Text.new
+      end
+    end
+
+    class << self
+      def config
+        @config ||= Configuration.new
+      end
+
+      def configure
+        yield config
+      end
+    end
+  end
+end
+
+require "test_prof/tps_prof/rspec" if TestProf.rspec?
+# TODO: Minitest support
+# require "test_prof/tps_prof/minitest" if TestProf.minitest?

data/lib/test_prof/vernier.rb

@@ -19,7 +19,7 @@ module TestProf
   module Vernier
     # Vernier configuration
     class Configuration
-      attr_accessor :mode, :target, :interval
+      attr_accessor :mode, :target, :interval, :hooks
 
       def initialize
         @mode = ENV.fetch("TEST_VERNIER_MODE", :wall).to_sym
@@ -27,6 +27,7 @@ module TestProf
 
         sample_interval = ENV["TEST_VERNIER_INTERVAL"].to_i
         @interval = (sample_interval > 0) ? sample_interval : nil
+        @hooks = ENV["TEST_VERNIER_HOOKS"]&.split(",")&.map { |hook| hook.strip.to_sym }
       end
 
       def boot?
@@ -82,6 +83,7 @@ module TestProf
         options = {}
 
         options[:interval] = config.interval if config.interval
+        options[:hooks] = config.hooks if config.hooks
 
         if block_given?
           options[:mode] = config.mode

data/lib/test_prof/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TestProf
-  VERSION = "1.3.0"
+  VERSION = "1.4.4"
 end

data/lib/test_prof.rb

@@ -12,5 +12,6 @@ require "test_prof/factory_prof"
 require "test_prof/memory_prof"
 require "test_prof/rspec_stamp"
 require "test_prof/tag_prof"
+require "test_prof/tps_prof"
 require "test_prof/rspec_dissect" if TestProf.rspec?
 require "test_prof/factory_all_stub"