derailed_benchmarks 1.4.2 → 1.8.0

data/lib/derailed_benchmarks/git_switch_project.rb

@@ -0,0 +1 @@
+

data/lib/derailed_benchmarks/load_tasks.rb

@@ -30,7 +30,7 @@ namespace :perf do
       DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
     end
 
-    if  ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
+    if !ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
       if defined? ActiveRecord::Tasks::DatabaseTasks
         ActiveRecord::Tasks::DatabaseTasks.create_current
       else # Rails 3.2
@@ -39,7 +39,14 @@ namespace :perf do
 
       ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
       ActiveRecord::Migration.verbose         = true
-      ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+
+      if Rails.version.start_with? '6'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrate
+      elsif Rails.version.start_with? '5.2'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrate
+      else
+        ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+      end
     end
 
     DERAILED_APP.config.consider_all_requests_local = true
@@ -135,4 +142,4 @@ namespace :perf do
       WARM_COUNT.times { call_app }
     end
   end
-end
+end

data/lib/derailed_benchmarks/require_tree.rb

@@ -40,7 +40,7 @@ module DerailedBenchmarks
     end
 
     def to_string
-      str = +"#{name}: #{cost.round(4)} MiB"
+      str = String.new("#{name}: #{cost.round(4)} MiB")
       if parent && REQUIRED_BY[self.name.to_s]
         names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
         if names.any?

data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb}

@@ -16,23 +16,34 @@ module DerailedBenchmarks
   #  x.average # => 10.5
   #  x.name    # => "muhfile"
   class StatsForFile
-    attr_reader :name, :values, :desc, :time
+    attr_reader :name, :values, :desc, :time, :short_sha
 
-    def initialize(file:, name:, desc: "", time: )
+    def initialize(file:, name:, desc: "", time: , short_sha: nil)
       @file = Pathname.new(file)
       FileUtils.touch(@file)
 
       @name = name
       @desc = desc
       @time = time
+      @short_sha = short_sha
     end
 
     def call
       load_file!
+      return if values.empty?
 
+      @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
       @average = values.inject(:+) / values.length
     end
 
+    def empty?
+      values.empty?
+    end
+
+    def median
+      @median.to_f
+    end
+
     def average
       @average.to_f
     end
@@ -47,6 +58,8 @@ module DerailedBenchmarks
           raise e, "Problem with file #{@file.inspect}:\n#{@file.read}\n#{e.message}"
         end
       end
+
+      values.sort!
       values.freeze
     end
   end

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -2,6 +2,9 @@
 
 require 'bigdecimal'
 require 'statistics'
+require 'stringio'
+require 'mini_histogram'
+require 'mini_histogram/plot'
 
 module DerailedBenchmarks
   # A class used to read several benchmark files
@@ -26,14 +29,28 @@ module DerailedBenchmarks
     FORMAT = "%0.4f"
     attr_reader :stats, :oldest, :newest
 
-    def initialize(hash)
+    def initialize(input)
       @files = []
 
-      hash.each do |branch, info_hash|
-        file = info_hash.fetch(:file)
-        desc = info_hash.fetch(:desc)
-        time = info_hash.fetch(:time)
-        @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
+      if input.is_a?(Hash)
+        hash = input
+        hash.each do |branch, info_hash|
+          file = info_hash.fetch(:file)
+          desc = info_hash.fetch(:desc)
+          time = info_hash.fetch(:time)
+          short_sha = info_hash["short_sha"]
+          @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
+        end
+      else
+        input.each do |commit|
+          @files << StatsForFile.new(
+            file: commit.file,
+            desc: commit.desc,
+            time: commit.time,
+            name: commit.ref,
+            short_sha: commit.short_sha
+          )
+        end
       end
       @files.sort_by! { |f| f.time }
       @oldest = @files.first
@@ -42,14 +59,27 @@ module DerailedBenchmarks
 
     def call
       @files.each(&:call)
-      @stats = statistical_test
+
+      return self if @files.detect(&:empty?)
+
+      stats_95 = statistical_test(confidence: 95)
+
+      # If default check is good, see if we also pass a more rigorous test
+      # if so, then use the more rigourous test
+      if stats_95[:alternative]
+        stats_99 = statistical_test(confidence: 99)
+        @stats = stats_99 if stats_99[:alternative]
+      end
+      @stats ||= stats_95
+
       self
     end
 
-    def statistical_test(series_1=oldest.values, series_2=newest.values)
+    def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
       StatisticalTest::KSTest.two_samples(
         group_one: series_1,
-        group_two: series_2
+        group_two: series_2,
+        alpha: (100 - confidence) / 100.0
       )
     end
 
@@ -66,18 +96,51 @@ module DerailedBenchmarks
     end
 
     def x_faster
-      FORMAT % (oldest.average/newest.average).to_f
+      (oldest.median/newest.median).to_f
+    end
+
+    def faster?
+      newest.median < oldest.median
     end
 
     def percent_faster
-      FORMAT % (((oldest.average - newest.average) / oldest.average).to_f  * 100)
+      (((oldest.median - newest.median) / oldest.median).to_f  * 100)
     end
 
     def change_direction
-      newest.average < oldest.average ? "FASTER" : "SLOWER"
+      if faster?
+        "FASTER 🚀🚀🚀"
+      else
+        "SLOWER 🐢🐢🐢"
+      end
+    end
+
+    def align
+      " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
     end
 
-    def banner(io = Kernel)
+    def histogram(io = $stdout)
+      newest_histogram = MiniHistogram.new(newest.values)
+      oldest_histogram = MiniHistogram.new(oldest.values)
+      MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
+
+      {newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
+        plot = histogram.plot(
+          title: "\n#{' ' * 18 }Histogram - [#{report.short_sha || report.name}] #{report.desc.inspect}",
+          ylabel: "Time (s)",
+          xlabel: "# of runs in range"
+        )
+
+        plot.render(io)
+        io.puts
+      end
+
+      io.puts
+    end
+
+    def banner(io = $stdout)
+      return if @files.detect(&:empty?)
+
       io.puts
       if significant?
         io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
@@ -85,19 +148,23 @@ module DerailedBenchmarks
         io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
       end
       io.puts
-      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.average} seconds)"
+      io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
       io.puts "  #{change_direction} by:"
-      io.puts "    #{x_faster}x [older/newer]"
-      io.puts "    #{percent_faster}\% [(older - newer) / older * 100]"
-      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.average} seconds)"
+      io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
+      io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
+      io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"
       io.puts
       io.puts "Test type: Kolmogorov Smirnov"
+      io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
       io.puts "Is significant? (max > critical): #{significant?}"
       io.puts "D critical: #{d_critical}"
       io.puts "D max: #{d_max}"
+
+      histogram(io)
+
       io.puts
     end
   end

data/lib/derailed_benchmarks/tasks.rb

@@ -1,6 +1,12 @@
 require_relative 'load_tasks'
 
 namespace :perf do
+  desc "runs the performance test against two most recent commits of the current app"
+  task :app do
+    ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
+    Rake::Task["perf:library"].invoke
+  end
+
   desc "runs the same test against two different branches for statistical comparison"
   task :library do
     begin
@@ -11,92 +17,65 @@ namespace :perf do
       script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
 
       if ENV["DERAILED_PATH_TO_LIBRARY"]
-        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
+        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"].chomp
       else
         library_dir = DerailedBenchmarks.rails_path_on_disk
       end
+      library_dir = Pathname.new(library_dir)
 
-      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
-
-      # Use either the explicit SHAs when present or grab last two SHAs from commit history
-      # if only one SHA is given, then use it and the last SHA from commit history
-      branch_names = []
-      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
-      if branch_names.length < 2
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
-
-          branches = run!('git log --format="%H" -n 2').chomp.split($/)
-          if branch_names.empty?
-            branch_names = branches
-          else
-            branches.shift
-            branch_names << branches.shift
-          end
-        end
-      end
-
-      current_library_branch = ""
-      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
-
-      out_dir = Pathname.new("tmp/library_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
+      out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
       out_dir.mkpath
 
-      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
-      branch_info = {}
-      branch_to_sha = {}
+      ref_string = ENV["SHAS_TO_TEST"] || ENV["REFS_TO_TEST"] || ""
 
-      branches_to_test.each do |branch, file|
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch}'")
-          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
-          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
-          short_sha   = run!("git rev-parse --short HEAD").strip
-          branch_to_sha[branch] = short_sha
+      project = DerailedBenchmarks::Git::SwitchProject.new(
+        path: library_dir,
+        ref_array: ref_string.split(","),
+        log_dir: out_dir
+      )
 
-          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
-        end
-        run!("#{script}")
-      end
+      stats = DerailedBenchmarks::StatsFromDir.new(project.commits)
 
+      # Advertise branch names early to make sure people know what they're testing
       puts
       puts
-      branches_to_test.each.with_index do |(branch, _), i|
-        short_sha = branch_to_sha[branch]
-        desc      = branch_info[short_sha][:desc]
-        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
+      project.commits.each_with_index do |commit, i|
+        puts "Testing #{i + 1}: #{commit.short_sha}: #{commit.description}"
       end
       puts
       puts
 
-      raise "SHAs to test must be different" if branch_info.length == 1
-      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-      ENV["DERAILED_STOP_VALID_COUNT"] ||= "50"
-      stop_valid_count = Integer(ENV["DERAILED_STOP_VALID_COUNT"])
-
-      times_significant = 0
-      DERAILED_SCRIPT_COUNT.times do |i|
-        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
-        branches_to_test.each do |branch, file|
-          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
-          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
+      project.restore_branch_on_return do
+        DERAILED_SCRIPT_COUNT.times do |i|
+          puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+          project.commits.each do |commit|
+            commit.checkout!
+
+            output = run!("#{script} 2>&1")
+            commit.log.open("a") {|f| f.puts output.lines.last }
+          end
+
+          if (i % 50).zero?
+            puts "Intermediate result"
+            stats.call.banner
+            puts "Continuing execution"
+          end
         end
-        times_significant += 1 if i >= 2 && stats.call.significant?
-        break if stop_valid_count != 0 && times_significant == stop_valid_count
       end
 
     ensure
-      if library_dir && current_library_branch
-        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{current_library_branch}'")
+      if stats
+        stats.call.banner
+
+        result_file = out_dir.join("results.txt")
+        File.open(result_file, "w") do |f|
+          stats.banner(f)
         end
-      end
 
-      stats.call.banner if stats
+        puts "Output: #{result_file.to_s}"
+      end
     end
-   end
-  
+  end
 
   desc "hits the url TEST_COUNT times"
   task :test => [:setup] do

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.4.2"
+  VERSION = "1.8.0"
 end

data/test/derailed_benchmarks/git_switch_project_test.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class GitSwitchProjectTest < ActiveSupport::TestCase
+  test "tells me when it's not pointing at a git project" do
+    exception = assert_raises {
+      DerailedBenchmarks::Git::SwitchProject.new(path: "/dev/null")
+    }
+    assert_includes(exception.message, '.git directory')
+  end
+
+  test "dirty gemspec cleaning" do
+    Dir.mktmpdir do |dir|
+      run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
+      run!("cd #{dir} && echo lol > foo.gemspec && git add .")
+
+      io = StringIO.new
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, io: io)
+
+      assert project.dirty?
+      refute project.clean?
+
+      project.restore_branch_on_return do
+        project.commits.map(&:checkout!)
+      end
+
+      assert_includes io.string, "Bundler modifies gemspec files"
+      assert_includes io.string, "Applying stash"
+    end
+  end
+
+  test "works on a git repo" do
+    Dir.mktmpdir do |dir|
+      run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
+
+      # finds shas when none given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir)
+
+      assert_equal ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "da748a59340be8b950e7bbbfb32077eb67d70c3c"], project.commits.map(&:ref)
+      first_commit = project.commits.first
+
+      assert_equal "CI test support", first_commit.description
+      assert_equal "6e64296", first_commit.short_sha
+      assert_equal "/dev/null/6e642963acec0ff64af51bd6fba8db3c4176ed6e.bench.txt", first_commit.log.to_s
+      assert_equal DateTime.parse("Tue, 14 Apr 2020 13:26:03 -0500"), first_commit.time
+
+      assert_equal "mybranch", project.current_branch_or_sha
+
+      # Finds shas when 1 is given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c"])
+
+      assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "5c09f748957d2098182762004adee27d1ff83160"], project.commits.map(&:ref)
+
+
+      # Returns correct refs if given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"])
+
+      assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"], project.commits.map(&:ref)
+
+      first_commit = project.commits.first
+
+      first_commit.checkout!
+
+      assert_equal first_commit.short_sha, project.current_branch_or_sha
+
+      # Test restore_branch_on_return
+      project.restore_branch_on_return(quiet: true) do
+        project.commits.last.checkout!
+
+        assert_equal project.commits.last.short_sha, project.current_branch_or_sha
+      end
+
+      assert_equal project.commits.first.short_sha, project.current_branch_or_sha
+
+      exception = assert_raise {
+        DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "mybranch"])
+      }
+
+      assert_includes(exception.message, 'Duplicate SHA resolved "6e64296"')
+    end
+  end
+end