derailed_benchmarks 1.4.3 → 1.8.1

data/lib/derailed_benchmarks/git_switch_project.rb

@@ -0,0 +1 @@
+

data/lib/derailed_benchmarks/load_tasks.rb

@@ -30,7 +30,7 @@ namespace :perf do
       DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
     end
 
-    if  ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
+    if !ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
       if defined? ActiveRecord::Tasks::DatabaseTasks
         ActiveRecord::Tasks::DatabaseTasks.create_current
       else # Rails 3.2
@@ -39,7 +39,14 @@ namespace :perf do
 
       ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
       ActiveRecord::Migration.verbose         = true
-      ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+
+      if Rails.version.start_with? '6'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrate
+      elsif Rails.version.start_with? '5.2'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrate
+      else
+        ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+      end
     end
 
     DERAILED_APP.config.consider_all_requests_local = true
@@ -103,7 +110,7 @@ namespace :perf do
           STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
 
           FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+          File.open("tmp/fail.html", "w+") {|f| f.write response }
 
           `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
 
@@ -135,4 +142,4 @@ namespace :perf do
       WARM_COUNT.times { call_app }
     end
   end
-end
+end

data/lib/derailed_benchmarks/require_tree.rb

@@ -13,6 +13,16 @@ module DerailedBenchmarks
     def initialize(name)
       @name     = name
       @children = {}
+      @cost = 0
+    end
+
+    def self.reset!
+      REQUIRED_BY.clear
+      if defined?(Kernel::REQUIRE_STACK)
+        Kernel::REQUIRE_STACK.clear
+
+        Kernel::REQUIRE_STACK.push(TOP_REQUIRE)
+      end
     end
 
     def <<(tree)
@@ -40,7 +50,7 @@ module DerailedBenchmarks
     end
 
     def to_string
-      str = +"#{name}: #{cost.round(4)} MiB"
+      str = String.new("#{name}: #{cost.round(4)} MiB")
       if parent && REQUIRED_BY[self.name.to_s]
         names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
         if names.any?

data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb}

@@ -16,24 +16,30 @@ module DerailedBenchmarks
   #  x.average # => 10.5
   #  x.name    # => "muhfile"
   class StatsForFile
-    attr_reader :name, :values, :desc, :time
+    attr_reader :name, :values, :desc, :time, :short_sha
 
-    def initialize(file:, name:, desc: "", time: )
+    def initialize(file:, name:, desc: "", time: , short_sha: nil)
       @file = Pathname.new(file)
       FileUtils.touch(@file)
 
       @name = name
       @desc = desc
       @time = time
+      @short_sha = short_sha
     end
 
     def call
       load_file!
+      return if values.empty?
 
       @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
       @average = values.inject(:+) / values.length
     end
 
+    def empty?
+      values.empty?
+    end
+
     def median
       @median.to_f
     end

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -2,6 +2,9 @@
 
 require 'bigdecimal'
 require 'statistics'
+require 'stringio'
+require 'mini_histogram'
+require 'mini_histogram/plot'
 
 module DerailedBenchmarks
   # A class used to read several benchmark files
@@ -26,14 +29,28 @@ module DerailedBenchmarks
     FORMAT = "%0.4f"
     attr_reader :stats, :oldest, :newest
 
-    def initialize(hash)
+    def initialize(input)
       @files = []
 
-      hash.each do |branch, info_hash|
-        file = info_hash.fetch(:file)
-        desc = info_hash.fetch(:desc)
-        time = info_hash.fetch(:time)
-        @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
+      if input.is_a?(Hash)
+        hash = input
+        hash.each do |branch, info_hash|
+          file = info_hash.fetch(:file)
+          desc = info_hash.fetch(:desc)
+          time = info_hash.fetch(:time)
+          short_sha = info_hash["short_sha"]
+          @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
+        end
+      else
+        input.each do |commit|
+          @files << StatsForFile.new(
+            file: commit.file,
+            desc: commit.desc,
+            time: commit.time,
+            name: commit.ref,
+            short_sha: commit.short_sha
+          )
+        end
       end
       @files.sort_by! { |f| f.time }
       @oldest = @files.first
@@ -42,14 +59,27 @@ module DerailedBenchmarks
 
     def call
       @files.each(&:call)
-      @stats = statistical_test
+
+      return self if @files.detect(&:empty?)
+
+      stats_95 = statistical_test(confidence: 95)
+
+      # If default check is good, see if we also pass a more rigorous test
+      # if so, then use the more rigourous test
+      if stats_95[:alternative]
+        stats_99 = statistical_test(confidence: 99)
+        @stats = stats_99 if stats_99[:alternative]
+      end
+      @stats ||= stats_95
+
       self
     end
 
-    def statistical_test(series_1=oldest.values, series_2=newest.values)
+    def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
       StatisticalTest::KSTest.two_samples(
         group_one: series_1,
-        group_two: series_2
+        group_two: series_2,
+        alpha: (100 - confidence) / 100.0
       )
     end
 
@@ -86,10 +116,31 @@ module DerailedBenchmarks
     end
 
     def align
-      " " * (("%i" % percent_faster).length - ("%i" % x_faster).length)
+      " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
+    end
+
+    def histogram(io = $stdout)
+      newest_histogram = MiniHistogram.new(newest.values)
+      oldest_histogram = MiniHistogram.new(oldest.values)
+      MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
+
+      {newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
+        plot = histogram.plot(
+          title: "\n#{' ' * 18 }Histogram - [#{report.short_sha || report.name}] #{report.desc.inspect}",
+          ylabel: "Time (s)",
+          xlabel: "# of runs in range"
+        )
+
+        plot.render(io)
+        io.puts
+      end
+
+      io.puts
     end
 
-    def banner(io = Kernel)
+    def banner(io = $stdout)
+      return if @files.detect(&:empty?)
+
       io.puts
       if significant?
         io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
@@ -97,19 +148,23 @@ module DerailedBenchmarks
         io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
       end
       io.puts
-      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
+      io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
       io.puts "  #{change_direction} by:"
       io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
       io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
-      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
+      io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"
       io.puts
       io.puts "Test type: Kolmogorov Smirnov"
+      io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
       io.puts "Is significant? (max > critical): #{significant?}"
       io.puts "D critical: #{d_critical}"
       io.puts "D max: #{d_max}"
+
+      histogram(io)
+
       io.puts
     end
   end

data/lib/derailed_benchmarks/tasks.rb

@@ -1,6 +1,12 @@
 require_relative 'load_tasks'
 
 namespace :perf do
+  desc "runs the performance test against two most recent commits of the current app"
+  task :app do
+    ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
+    Rake::Task["perf:library"].invoke
+  end
+
   desc "runs the same test against two different branches for statistical comparison"
   task :library do
     begin
@@ -11,92 +17,57 @@ namespace :perf do
       script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
 
       if ENV["DERAILED_PATH_TO_LIBRARY"]
-        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
+        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"].chomp
       else
         library_dir = DerailedBenchmarks.rails_path_on_disk
       end
-
-      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
-
-      # Use either the explicit SHAs when present or grab last two SHAs from commit history
-      # if only one SHA is given, then use it and the last SHA from commit history
-      branch_names = []
-      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
-      if branch_names.length < 2
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
-
-          branches = run!('git log --format="%H" -n 2').chomp.split($/)
-          if branch_names.empty?
-            branch_names = branches
-          else
-            branches.shift
-            branch_names << branches.shift
-          end
-        end
-      end
-
-      current_library_branch = ""
-      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
+      library_dir = Pathname.new(library_dir)
 
       out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
       out_dir.mkpath
 
-      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
-      branch_info = {}
-      branch_to_sha = {}
+      ref_string = ENV["SHAS_TO_TEST"] || ENV["REFS_TO_TEST"] || ""
 
-      branches_to_test.each do |branch, file|
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch}'")
-          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
-          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
-          short_sha   = run!("git rev-parse --short HEAD").strip
-          branch_to_sha[branch] = short_sha
+      project = DerailedBenchmarks::Git::SwitchProject.new(
+        path: library_dir,
+        ref_array: ref_string.split(","),
+        log_dir: out_dir
+      )
 
-          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
-        end
-        run!("#{script}")
-      end
+      stats = DerailedBenchmarks::StatsFromDir.new(project.commits)
 
+      # Advertise branch names early to make sure people know what they're testing
       puts
       puts
-      branches_to_test.each.with_index do |(branch, _), i|
-        short_sha = branch_to_sha[branch]
-        desc      = branch_info[short_sha][:desc]
-        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
+      project.commits.each_with_index do |commit, i|
+        puts "Testing #{i + 1}: #{commit.short_sha}: #{commit.description}"
       end
       puts
       puts
 
-      raise "SHAs to test must be different" if branch_info.length == 1
-      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-      ENV["DERAILED_STOP_VALID_COUNT"] ||= "50"
-      stop_valid_count = Integer(ENV["DERAILED_STOP_VALID_COUNT"])
-
-      times_significant = 0
-      DERAILED_SCRIPT_COUNT.times do |i|
-        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
-        branches_to_test.each do |branch, file|
-          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
-          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
-        end
-        times_significant += 1 if i >= 2 && stats.call.significant?
-        break if stop_valid_count != 0 && times_significant == stop_valid_count
-      end
+      project.restore_branch_on_return do
+        DERAILED_SCRIPT_COUNT.times do |i|
+          puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+          project.commits.each do |commit|
+            commit.checkout!
 
-    ensure
-      if library_dir && current_library_branch
-        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{current_library_branch}'")
+            output = run!("#{script} 2>&1")
+            commit.log.open("a") {|f| f.puts output.lines.last }
+          end
+
+          if (i % 50).zero?
+            puts "Intermediate result"
+            stats.call.banner
+            puts "Continuing execution"
+          end
         end
       end
 
+    ensure
       if stats
         stats.call.banner
 
-        result_file = out_dir + "results.txt"
+        result_file = out_dir.join("results.txt")
         File.open(result_file, "w") do |f|
           stats.banner(f)
         end

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.4.3"
+  VERSION = "1.8.1"
 end

data/test/derailed_benchmarks/core_ext/kernel_require_test.rb

@@ -3,7 +3,6 @@
 require 'test_helper'
 
 class KernelRequireTest < ActiveSupport::TestCase
-
   setup do
     require 'derailed_benchmarks/core_ext/kernel_require'
     GC.disable
@@ -11,23 +10,83 @@ class KernelRequireTest < ActiveSupport::TestCase
 
   teardown do
     GC.enable
+    DerailedBenchmarks::RequireTree.reset!
+  end
+
+  test "profiles load" do
+    in_fork do
+      require fixtures_dir("require/load_parent.rb")
+
+      parent = assert_node_in_parent("load_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("load_child.rb", parent)
+    end
+  end
+
+  test "profiles autoload" do
+    skip if RUBY_VERSION.start_with?("2.2") # Fails on CI, I can't install Ruby 2.2 locally to debug https://stackoverflow.com/questions/63926460/install-ruby-2-2-on-mac-osx-catalina-with-ruby-install, https://github.com/postmodern/ruby-install/issues/375
+
+    in_fork do
+      require fixtures_dir("require/autoload_parent.rb")
+      parent = assert_node_in_parent("autoload_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("autoload_child.rb", parent)
+    end
   end
 
+  test "core extension profiles useage" do
+    in_fork do
+      require fixtures_dir("require/parent_one.rb")
+      parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
+      assert_node_in_parent("child_one.rb", parent)
+      child_two = assert_node_in_parent("child_two.rb", parent)
+      assert_node_in_parent("relative_child", parent)
+      assert_node_in_parent("relative_child_two", parent)
+      assert_node_in_parent("raise_child.rb", child_two)
+    end
+  end
+
+  # Checks to see that the given file name is present in the
+  # parent tree node and that the memory of that file
+  # is less than the parent (since the parent should include itself
+  # plus its children)
+  #
+  # Returns the child node
   def assert_node_in_parent(file_name, parent)
     file = fixtures_dir(File.join("require", file_name))
     node = parent[file]
-    assert node,                    "Expected:\n#{parent.children}\nto include:\n#{file.inspect}"
-    assert node.cost < parent.cost, "Expected:\n#{node.inspect}\nto cost less than:\n#{parent.inspect}" unless parent == TOP_REQUIRE
+    assert node, "Expected: #{parent.name} to include: #{file.to_s} but it did not.\nChildren: #{parent.children.map(&:name).map(&:to_s)}"
+    unless parent == TOP_REQUIRE
+      assert node.cost < parent.cost, "Expected: #{node.name.inspect} (#{node.cost}) to cost less than: #{parent.name.inspect} (#{parent.cost})"
+    end
     node
   end
 
-  test "core extension profiles useage" do
-    require fixtures_dir("require/parent_one.rb")
-    parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
-    assert_node_in_parent("child_one.rb", parent)
-    child_two = assert_node_in_parent("child_two.rb", parent)
-    assert_node_in_parent("relative_child", parent)
-    assert_node_in_parent("relative_child_two", parent)
-    assert_node_in_parent("raise_child.rb", child_two)
+  # Used to get semi-clean process memory
+  # It would be better to run the requires in a totally different process
+  # but...that would take engineering
+  #
+  # If I was going to do that, I would find a way to serialize RequireTree
+  # into a json structure with file names and costs, run the script
+  # dump the json to a file, then in this process read the file and
+  # run assertions
+  def in_fork
+    Tempfile.create("stdout") do |tmp_file|
+      pid = fork do
+        $stdout.reopen(tmp_file, "w")
+        $stderr.reopen(tmp_file, "w")
+        $stdout.sync = true
+        $stderr.sync = true
+        yield
+        Kernel.exit!(0) # needed for https://github.com/seattlerb/minitest/pull/683
+      end
+      Process.waitpid(pid)
+
+      if $?.success?
+        print File.read(tmp_file)
+      else
+        raise File.read(tmp_file)
+      end
+    end
   end
 end