test-prof 1.2.2 → 1.3.0

data/lib/test_prof/memory_prof/tracker/linked_list.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module TestProf
+  module MemoryProf
+    class Tracker
+      # LinkedList is a linked list that track memory usage for individual examples/groups.
+      # A list node (`LinkedListNode`) represents an example/group and its memory usage info:
+      #
+      # * memory_at_start - the amount of memory at the start of an example/group
+      # * memory_at_finish - the amount of memory at the end of an example/group
+      # * nested_memory - the amount of memory allocated by examples/groups defined inside a group
+      # * previous - a link to the previous node
+      #
+      # Each node has a link to its previous node, and the head node points to the current example/group.
+      # If we picture a linked list as a tree with root being the top level group and leaves being
+      # current examples/groups, then the head node will always point to a leaf in that tree.
+      #
+      # For example, if we have the following spec:
+      #
+      #   describe Question do
+      #     decribe "#publish" do
+      #       context "when not published" do
+      #         it "makes the question visible" do
+      #           ...
+      #         end
+      #       end
+      #     end
+      #   end
+      #
+      # At the moment when rspec is executing the example, the list has the following structure
+      # (^ denotes the head node):
+      #
+      #    ^"makes the question visible" ->  "when not published" -> "#publish" -> Question
+      #
+      # LinkedList supports two method for working with it:
+      #
+      #  * add_node – adds a node to the beginig of the list. At this point an example or group
+      #    has started and we track how much memory has already been used.
+      #  * remove_node – removes and returns the head node from the list. It means that the node
+      #    example/group has finished and it is time to calculate its memory usage.
+      #
+      # When we remove a node we add its total_memory to the previous node.nested_memory, thus
+      # gradually tracking the amount of memory used by nested examples inside a group.
+      #
+      # In the example above, after we remove the node "makes the question visible", we add its total
+      # memory usage to nested_memory of the "when not published" node. If the "when not published"
+      # group contains other examples or sub-groups, their total_memory will also be added to
+      # "when not published" nested_memory. So when the group finishes we will have the total amount
+      # of memory used by its nested examples/groups, and thus we will be able to calculate the memory
+      # used by hooks and other code inside a group by subtracting nested_memory from total_memory.
+      class LinkedList
+        attr_reader :head
+
+        def initialize(memory_at_start)
+          add_node(:total, memory_at_start)
+        end
+
+        def add_node(item, memory_at_start)
+          @head = LinkedListNode.new(
+            item: item,
+            previous: head,
+            memory_at_start: memory_at_start
+          )
+        end
+
+        def remove_node(item, memory_at_finish)
+          return if head.item != item
+          head.finish(memory_at_finish)
+
+          current = head
+          @head = head.previous
+
+          current
+        end
+      end
+
+      class LinkedListNode
+        attr_reader :item, :previous, :memory_at_start, :memory_at_finish, :nested_memory
+
+        def initialize(item:, memory_at_start:, previous:)
+          @item = item
+          @previous = previous
+
+          @memory_at_start = memory_at_start || 0
+          @memory_at_finish = nil
+          @nested_memory = 0
+        end
+
+        def total_memory
+          return 0 if memory_at_finish.nil?
+          # It seems that on Windows Minitest may release a lot of memory to
+          # the OS when it finishes and executes #report, leading to memory_at_finish
+          # being less than memory_at_start. In this case we return nested_memory
+          # which does not account for the memory used in `after` hooks, but it
+          # is better than nothing.
+          return nested_memory if memory_at_start > memory_at_finish
+
+          memory_at_finish - memory_at_start
+        end
+
+        def hooks_memory
+          total_memory - nested_memory
+        end
+
+        def finish(memory_at_finish)
+          @memory_at_finish = memory_at_finish
+
+          previous&.add_nested(self)
+        end
+
+        protected
+
+        def add_nested(node)
+          @nested_memory += node.total_memory
+        end
+      end
+    end
+  end
+end

data/lib/test_prof/memory_prof/tracker/rss_tool.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+
+module TestProf
+  module MemoryProf
+    class Tracker
+      module RssTool
+        class ProcFS
+          def initialize
+            @statm = File.open("/proc/#{$$}/statm", "r")
+            @page_size = get_page_size
+          end
+
+          def track
+            @statm.seek(0)
+            @statm.gets.split(/\s/)[1].to_i * @page_size
+          end
+
+          private
+
+          def get_page_size
+            [
+              -> {
+                require "etc"
+                Etc.sysconf(Etc::SC_PAGE_SIZE)
+              },
+              -> { `getconf PAGE_SIZE`.to_i },
+              -> { 0x1000 }
+            ].each do |strategy|
+              page_size = begin
+                strategy.call
+              rescue
+                next
+              end
+              return page_size
+            end
+          end
+        end
+
+        class PS
+          def track
+            `ps -o rss -p #{$$}`.strip.split.last.to_i * 1024
+          end
+        end
+
+        class GetProcess
+          def track
+            command.strip.split.last.to_i
+          end
+
+          private
+
+          def command
+            `powershell -Command "Get-Process -Id #{$$} | select WS"`
+          end
+        end
+
+        TOOLS = {
+          linux: ProcFS,
+          macosx: PS,
+          unix: PS,
+          windows: GetProcess
+        }.freeze
+
+        class << self
+          def tool
+            TOOLS[os_type]&.new
+          end
+
+          def os_type
+            case RbConfig::CONFIG["host_os"]
+            when /linux/
+              :linux
+            when /darwin|mac os/
+              :macosx
+            when /solaris|bsd/
+              :unix
+            when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+              :windows
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/test_prof/memory_prof/tracker.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "test_prof/memory_prof/tracker/linked_list"
+require "test_prof/memory_prof/tracker/rss_tool"
+
+module TestProf
+  module MemoryProf
+    # Tracker is responsible for tracking memory usage and determining
+    # the top n examples and groups. There are two types of trackers:
+    # AllocTracker and RssTracker.
+    #
+    # A tracker consists of four main parts:
+    #  * list - a linked list that is being used to track memmory for individual groups/examples.
+    #    list is an instance of LinkedList (for more info see tracker/linked_list.rb)
+    #  * examples – the top n examples, an instance of Utils::SizedOrderedSet.
+    #  * groups – the top n groups, an instance of Utils::SizedOrderedSet.
+    #  * track - a method that fetches the amount of memory in use at a certain point.
+    class Tracker
+      attr_reader :top_count, :examples, :groups, :total_memory, :list
+
+      def initialize(top_count)
+        raise "Your Ruby Engine or OS is not supported" unless supported?
+
+        @top_count = top_count
+
+        @examples = Utils::SizedOrderedSet.new(top_count, sort_by: :memory)
+        @groups = Utils::SizedOrderedSet.new(top_count, sort_by: :memory)
+      end
+
+      def start
+        @list = LinkedList.new(track)
+      end
+
+      def finish
+        node = list.remove_node(:total, track)
+        @total_memory = node.total_memory
+      end
+
+      def example_started(example)
+        list.add_node(example, track)
+      end
+
+      def example_finished(example)
+        node = list.remove_node(example, track)
+        examples << {**example, memory: node.total_memory}
+      end
+
+      def group_started(group)
+        list.add_node(group, track)
+      end
+
+      def group_finished(group)
+        node = list.remove_node(group, track)
+        groups << {**group, memory: node.hooks_memory}
+      end
+    end
+
+    class AllocTracker < Tracker
+      def track
+        GC.stat[:total_allocated_objects]
+      end
+
+      def supported?
+        RUBY_ENGINE != "jruby"
+      end
+    end
+
+    class RssTracker < Tracker
+      def initialize(top_count)
+        @rss_tool = RssTool.tool
+
+        super
+      end
+
+      def track
+        @rss_tool.track
+      end
+
+      def supported?
+        !!@rss_tool
+      end
+    end
+  end
+end

data/lib/test_prof/memory_prof.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "test_prof/memory_prof/tracker"
+require "test_prof/memory_prof/printer"
+
+module TestProf
+  # MemoryProf can help in detecting test examples causing memory spikes.
+  # It supports two metrics: RSS and allocations.
+  #
+  # Example:
+  #
+  #   TEST_MEM_PROF='rss' rspec ...
+  #   TEST_MEM_PROF='alloc' rspec ...
+  #
+  # By default MemoryProf shows the top 5 examples and groups (for RSpec) but you can
+  # set how many items to display with `TEST_MEM_PROF_COUNT`:
+  #
+  #   TEST_MEM_PROF='rss' TEST_MEM_PROF_COUNT=10 rspec ...
+  #
+  # The examples block shows the amount of memory used by each example, and the groups
+  # block displays the memory allocated by other code defined in the groups. For example,
+  # RSpec groups may include heavy `before(:all)` (or `before_all`) setup blocks, so it is
+  # helpful to see which groups use the most amount of memory outside of their examples.
+
+  module MemoryProf
+    # MemoryProf configuration
+    class Configuration
+      attr_reader :mode, :top_count
+
+      def initialize
+        self.mode = ENV["TEST_MEM_PROF"]
+        self.top_count = ENV["TEST_MEM_PROF_COUNT"]
+      end
+
+      def mode=(value)
+        @mode = (value == "alloc") ? :alloc : :rss
+      end
+
+      def top_count=(value)
+        @top_count = value.to_i
+        @top_count = 5 unless @top_count.positive?
+      end
+    end
+
+    class << self
+      TRACKERS = {
+        alloc: AllocTracker,
+        rss: RssTracker
+      }.freeze
+
+      PRINTERS = {
+        alloc: AllocPrinter,
+        rss: RssPrinter
+      }.freeze
+
+      def config
+        @config ||= Configuration.new
+      end
+
+      def configure
+        yield config
+      end
+
+      def tracker
+        tracker = TRACKERS[config.mode]
+        tracker.new(config.top_count)
+      end
+
+      def printer(tracker)
+        printer = PRINTERS[config.mode]
+        printer.new(tracker)
+      end
+    end
+  end
+end
+
+require "test_prof/memory_prof/rspec" if TestProf.rspec?
+require "test_prof/memory_prof/minitest" if TestProf.minitest?

data/lib/test_prof/recipes/logging.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "test_prof"
+require "test_prof/core"
 
 module TestProf
   module Rails

data/lib/test_prof/recipes/minitest/sample.rb

@@ -6,9 +6,9 @@ module TestProf
     # Do not add these classes to resulted sample
     CORE_RUNNABLES = [
       Minitest::Test,
-      Minitest::Unit::TestCase,
+      defined?(Minitest::Unit::TestCase) ? Minitest::Unit::TestCase : nil,
       Minitest::Spec
-    ].freeze
+    ].compact.freeze
 
     class << self
       def suites

data/lib/test_prof/recipes/rspec/let_it_be.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "test_prof"
+require "test_prof/core"
 require "test_prof/recipes/rspec/before_all"
 
 module TestProf

data/lib/test_prof/ruby_prof.rb

@@ -53,7 +53,7 @@ module TestProf
       def initialize
         @printer = ENV["TEST_RUBY_PROF"].to_sym if PRINTERS.key?(ENV["TEST_RUBY_PROF"])
         @printer ||= ENV.fetch("TEST_RUBY_PROF_PRINTER", :flat).to_sym
-        @mode = ENV.fetch("TEST_RUBY_PROF_MODE", :wall).to_sym
+        @mode = ENV.fetch("TEST_RUBY_PROF_MODE", :wall).to_s
         @min_percent = 1
         @include_threads = false
         @exclude_common_methods = true
@@ -84,6 +84,22 @@ module TestProf
 
         [type, ::RubyProf.const_get(PRINTERS[type])]
       end
+
+      # Based on deprecated https://github.com/ruby-prof/ruby-prof/blob/fd3a5236a459586c5ca7ce4de506c1835129516a/lib/ruby-prof.rb#L36
+      def ruby_prof_mode
+        case mode
+        when "wall", "wall_time"
+          ::RubyProf::WALL_TIME
+        when "allocations"
+          ::RubyProf::ALLOCATIONS
+        when "memory"
+          ::RubyProf::MEMORY
+        when "process", "process_time"
+          ::RubyProf::PROCESS_TIME
+        else
+          ::RubyProf::WALL_TIME
+        end
+      end
     end
 
     # Wrapper over RubyProf profiler and printer
@@ -166,7 +182,7 @@ module TestProf
           log :warn, <<~MSG
             RubyProf is activated globally, you cannot generate per-example report.
 
-            Make sure you haven's set the TEST_RUBY_PROF environmental variable.
+            Make sure you haven't set the TEST_RUBY_PROF environmental variable.
           MSG
           return
         end
@@ -177,6 +193,7 @@ module TestProf
 
         options[:include_threads] = [Thread.current] unless
           config.include_threads?
+        options[:measure_mode] = config.ruby_prof_mode
 
         profiler = ::RubyProf::Profile.new(options)
         profiler.exclude_common_methods! if config.exclude_common_methods?
@@ -206,7 +223,6 @@ module TestProf
 
       def init_ruby_prof
         return @initialized if instance_variable_defined?(:@initialized)
-        ENV["RUBY_PROF_MEASURE_MODE"] = config.mode.to_s
         @initialized = TestProf.require(
           "ruby-prof",
           <<~MSG

data/lib/test_prof/stack_prof/rspec.rb

@@ -19,12 +19,13 @@ module TestProf
 
       def example_started(notification)
         return unless profile?(notification.example)
+
         notification.example.metadata[:sprof_report] = TestProf::StackProf.profile
       end
 
       def example_finished(notification)
         return unless profile?(notification.example)
-        return unless notification.example.metadata[:sprof_report] == false
+        return if notification.example.metadata[:sprof_report] == false
 
         TestProf::StackProf.dump(
           self.class.report_name_generator.call(notification.example)

data/lib/test_prof/stack_prof.rb

@@ -33,7 +33,7 @@ module TestProf
           if FORMATS.include?(ENV["TEST_STACK_PROF_FORMAT"])
             ENV["TEST_STACK_PROF_FORMAT"]
           else
-            "html"
+            "json"
           end
 
         sample_interval = ENV["TEST_STACK_PROF_INTERVAL"].to_i
@@ -81,9 +81,9 @@ module TestProf
       def profile(name = nil)
         if locked?
           log :warn, <<~MSG
-            StackProf is activated globally, you cannot generate per-example report.
+            StackProf has been already activated.
 
-            Make sure you haven's set the TEST_STACK_PROF environmental variable.
+            Make sure you have not set the TEST_STACK_PROF environmental variable.
           MSG
           return false
         end

data/lib/test_prof/utils/sized_ordered_set.rb

@@ -53,6 +53,10 @@ module TestProf
         data.size
       end
 
+      def empty?
+        size.zero?
+      end
+
       def to_a
         data.dup
       end

data/lib/test_prof/vernier/rspec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "test_prof/utils/rspec"
+
+module TestProf
+  module Vernier
+    # Reporter for RSpec to profile specific examples with Vernier
+    class Listener # :nodoc:
+      class << self
+        attr_accessor :report_name_generator
+      end
+
+      self.report_name_generator = Utils::RSpec.method(:example_to_filename)
+
+      NOTIFICATIONS = %i[
+        example_started
+        example_finished
+      ].freeze
+
+      def example_started(notification)
+        return unless profile?(notification.example)
+        notification.example.metadata[:vernier_collector] = TestProf::Vernier.profile
+      end
+
+      def example_finished(notification)
+        return unless profile?(notification.example)
+        return unless notification.example.metadata[:vernier_collector]
+
+        TestProf::Vernier.dump(
+          notification.example.metadata[:vernier_collector],
+          self.class.report_name_generator.call(notification.example)
+        )
+      end
+
+      private
+
+      def profile?(example)
+        example.metadata.key?(:vernier)
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    listener = TestProf::Vernier::Listener.new
+
+    config.reporter.register_listener(
+      listener, *TestProf::Vernier::Listener::NOTIFICATIONS
+    )
+  end
+end
+
+# Handle boot profiling
+RSpec.configure do |config|
+  config.append_before(:suite) do
+    TestProf::Vernier.dump(TestProf::Vernier.default_collector, "boot") if TestProf::Vernier.config.boot?
+  end
+end