derailed_benchmarks 1.5.0 → 2.0.0

data/lib/derailed_benchmarks/git/in_path.rb

@@ -0,0 +1,59 @@
+module DerailedBenchmarks
+  # A class for running commands in a git directory
+  #
+  # It's faster to check if we're already in that directory instead
+  # of having to `cd` into each time. https://twitter.com/schneems/status/1305196730170961920
+  #
+  # Example:
+  #
+  #   in_git_path = InGitPath.new(`bundle info heapy --path`.strip)
+  #   in_git_path.checkout!("f0f92b06156f2274021aa42f15326da041ee9009")
+  #   in_git_path.short_sha # => "f0f92b0"
+  class Git::InPath
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def description
+      run!("git log --oneline --format=%B -n 1 HEAD | head -n 1")
+    end
+
+    def short_sha
+      run!("git rev-parse --short HEAD")
+    end
+
+    def time_stamp_string
+      run!("git log -n 1 --pretty=format:%ci") # https://stackoverflow.com/a/25921837/147390
+    end
+
+    def branch
+      branch = run!("git rev-parse --abbrev-ref HEAD")
+      branch == "HEAD" ? nil : branch
+    end
+
+    def checkout!(ref)
+      run!("git checkout '#{ref}' 2>&1")
+    end
+
+    def time
+      DateTime.parse(time_stamp_string)
+    end
+
+    def run(cmd)
+      if Dir.pwd == path
+        out = `#{cmd}`.strip
+      else
+        out = `cd #{path} && #{cmd}`.strip
+      end
+      out
+    end
+
+    def run!(cmd)
+      out = run(cmd)
+      raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+      out
+    end
+  end
+end

data/lib/derailed_benchmarks/git/switch_project.rb

@@ -0,0 +1,128 @@
+module DerailedBenchmarks
+  class Git
+  end
+end
+require_relative "in_path.rb"
+require_relative "commit.rb"
+
+module DerailedBenchmarks
+  # Wraps two or more git commits in a specific location
+  #
+  # Returns an array of GitCommit objects that can be used to manipulate
+  # and checkout the repo
+  #
+  # Example:
+  #
+  #   `git clone https://sharpstone/default_ruby tmp/default_ruby`
+  #
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby")
+  #
+  # By default it will represent the last two commits:
+  #
+  #   project.commits.length # => 2
+  #
+  # You can pass in explicit REFs in an array:
+  #
+  #   ref_array = ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"]
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby", ref_array: ref_array)
+  #
+  #   puts project.commits.map(&:ref) == ref_array # => true
+  #
+  #
+  # It knows the current branch or sha:
+  #
+  #    `cd tmp/ruby && git checkout -b mybranch`
+  #    project.current_branch_or_sha #=> "mybranch"
+  #
+  # It can be used for safely wrapping checkouts to ensure the project returns to it's original branch:
+  #
+  #    project.restore_branch_on_return do
+  #      project.commits.first.checkout!
+  #      project.current_branch_or_sha # => "da748a593"
+  #    end
+  #
+  #    project.current_branch_or_sha # => "mybranch"
+  class Git::SwitchProject
+    attr_reader :commits
+
+    def initialize(path: , ref_array: [], io: STDOUT, log_dir: "/dev/null")
+      @path = Pathname.new(path)
+
+      @in_git_path = Git::InPath.new(@path.expand_path)
+
+      raise "Must be a path with a .git directory '#{@path}'" if !@path.join(".git").exist?
+      @io = io
+      @commits = []
+      log_dir = Pathname(log_dir)
+
+      expand_refs(ref_array).each do |ref|
+        restore_branch_on_return(quiet: true) do
+          @commits << Git::Commit.new(path: @path, ref: ref, log_dir: log_dir)
+        end
+      end
+
+      if (duplicate = @commits.group_by(&:short_sha).detect {|(k, v)| v.length > 1})
+        raise "Duplicate SHA resolved #{duplicate[0].inspect}: #{duplicate[1].map {|c| "'#{c.ref}' => '#{c.short_sha}'"}.join(", ") } at #{@path}"
+      end
+    end
+
+    def current_branch_or_sha
+      branch_or_sha = @in_git_path.branch
+      branch_or_sha ||= @in_git_path.short_sha
+      branch_or_sha
+    end
+
+    def dirty?
+      !clean?
+    end
+
+    # https://stackoverflow.com/a/3879077/147390
+    def clean?
+      @in_git_path.run("git diff-index --quiet HEAD --") && $?.success?
+    end
+
+    private def status(pattern: "*.gemspec")
+      @in_git_path.run("git status #{pattern}")
+    end
+
+    def restore_branch_on_return(quiet: false)
+      if dirty? && status.include?("gemspec")
+        dirty_gemspec = true
+        unless quiet
+          @io.puts "Working tree at #{@path} is dirty, stashing. This will be popped on return"
+          @io.puts "Bundler modifies gemspec files on git install, this is normal"
+          @io.puts "Original status:\n#{status}"
+        end
+        @in_git_path.run!("git stash")
+      end
+      branch_or_sha = self.current_branch_or_sha
+      yield
+    ensure
+      return unless branch_or_sha
+      @io.puts "Resetting git dir of '#{@path.to_s}' to #{branch_or_sha.inspect}" unless quiet
+
+      @in_git_path.checkout!(branch_or_sha)
+      if dirty_gemspec
+        out = @in_git_path.run!("git stash apply 2>&1")
+        @io.puts "Applying stash of '#{@path.to_s}':\n#{out}" unless quiet
+      end
+    end
+
+    # case ref_array.length
+    # when >= 2
+    #   returns original array
+    # when 1
+    #   returns the given ref plus the one before it
+    # when 0
+    #   returns the most recent 2 refs
+    private def expand_refs(ref_array)
+      return ref_array if ref_array.length >= 2
+
+      @in_git_path.checkout!(ref_array.first) if ref_array.first
+
+      branches_string = @in_git_path.run!("git log --format='%H' -n 2")
+      ref_array = branches_string.split($/)
+      return ref_array
+    end
+  end
+end

data/lib/derailed_benchmarks/git_switch_project.rb

@@ -0,0 +1 @@
+

data/lib/derailed_benchmarks/load_tasks.rb

@@ -30,7 +30,7 @@ namespace :perf do
       DERAILED_APP.initialize! unless DERAILED_APP.instance_variable_get(:@initialized)
     end
 
-    if  ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
+    if !ENV["DERAILED_SKIP_ACTIVE_RECORD"] && defined? ActiveRecord
       if defined? ActiveRecord::Tasks::DatabaseTasks
         ActiveRecord::Tasks::DatabaseTasks.create_current
       else # Rails 3.2
@@ -39,7 +39,14 @@ namespace :perf do
 
       ActiveRecord::Migrator.migrations_paths = DERAILED_APP.paths['db/migrate'].to_a
       ActiveRecord::Migration.verbose         = true
-      ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+
+      if Rails.version.start_with? '6'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths, ActiveRecord::SchemaMigration).migrate
+      elsif Rails.version.start_with? '5.2'
+        ActiveRecord::MigrationContext.new(ActiveRecord::Migrator.migrations_paths).migrate
+      else
+        ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, nil)
+      end
     end
 
     DERAILED_APP.config.consider_all_requests_local = true
@@ -103,7 +110,7 @@ namespace :perf do
           STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
 
           FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+          File.open("tmp/fail.html", "w+") {|f| f.write response }
 
           `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
 
@@ -135,4 +142,4 @@ namespace :perf do
       WARM_COUNT.times { call_app }
     end
   end
-end
+end

data/lib/derailed_benchmarks/require_tree.rb

@@ -13,6 +13,16 @@ module DerailedBenchmarks
     def initialize(name)
       @name     = name
       @children = {}
+      @cost = 0
+    end
+
+    def self.reset!
+      REQUIRED_BY.clear
+      if defined?(Kernel::REQUIRE_STACK)
+        Kernel::REQUIRE_STACK.clear
+
+        Kernel::REQUIRE_STACK.push(TOP_REQUIRE)
+      end
     end
 
     def <<(tree)
@@ -40,7 +50,7 @@ module DerailedBenchmarks
     end
 
     def to_string
-      str = +"#{name}: #{cost.round(4)} MiB"
+      str = String.new("#{name}: #{cost.round(4)} MiB")
       if parent && REQUIRED_BY[self.name.to_s]
         names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
         if names.any?

data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb}

@@ -16,24 +16,30 @@ module DerailedBenchmarks
   #  x.average # => 10.5
   #  x.name    # => "muhfile"
   class StatsForFile
-    attr_reader :name, :values, :desc, :time
+    attr_reader :name, :values, :desc, :time, :short_sha
 
-    def initialize(file:, name:, desc: "", time: )
+    def initialize(file:, name:, desc: "", time: , short_sha: nil)
       @file = Pathname.new(file)
       FileUtils.touch(@file)
 
       @name = name
       @desc = desc
       @time = time
+      @short_sha = short_sha
     end
 
     def call
       load_file!
+      return if values.empty?
 
       @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
       @average = values.inject(:+) / values.length
     end
 
+    def empty?
+      values.empty?
+    end
+
     def median
       @median.to_f
     end

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -2,6 +2,9 @@
 
 require 'bigdecimal'
 require 'statistics'
+require 'stringio'
+require 'mini_histogram'
+require 'mini_histogram/plot'
 
 module DerailedBenchmarks
   # A class used to read several benchmark files
@@ -26,14 +29,28 @@ module DerailedBenchmarks
     FORMAT = "%0.4f"
     attr_reader :stats, :oldest, :newest
 
-    def initialize(hash)
+    def initialize(input)
       @files = []
 
-      hash.each do |branch, info_hash|
-        file = info_hash.fetch(:file)
-        desc = info_hash.fetch(:desc)
-        time = info_hash.fetch(:time)
-        @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
+      if input.is_a?(Hash)
+        hash = input
+        hash.each do |branch, info_hash|
+          file = info_hash.fetch(:file)
+          desc = info_hash.fetch(:desc)
+          time = info_hash.fetch(:time)
+          short_sha = info_hash[:short_sha]
+          @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
+        end
+      else
+        input.each do |commit|
+          @files << StatsForFile.new(
+            file: commit.file,
+            desc: commit.desc,
+            time: commit.time,
+            name: commit.ref,
+            short_sha: commit.short_sha
+          )
+        end
       end
       @files.sort_by! { |f| f.time }
       @oldest = @files.first
@@ -43,6 +60,8 @@ module DerailedBenchmarks
     def call
       @files.each(&:call)
 
+      return self if @files.detect(&:empty?)
+
       stats_95 = statistical_test(confidence: 95)
 
       # If default check is good, see if we also pass a more rigorous test
@@ -100,7 +119,29 @@ module DerailedBenchmarks
       " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
     end
 
-    def banner(io = Kernel)
+    def histogram(io = $stdout)
+      dual_histogram = MiniHistogram.dual_plot do |a, b|
+        a.values = newest.values
+        a.options = {
+          title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
+          xlabel: "# of runs in range"
+        }
+        b.values = oldest.values
+        b.options = {
+          title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
+          xlabel: "# of runs in range"
+        }
+      end
+
+      io.puts
+      io.puts "Histograms (time ranges are in seconds):"
+      io.puts(dual_histogram)
+      io.puts
+    end
+
+    def banner(io = $stdout)
+      return if @files.detect(&:empty?)
+
       io.puts
       if significant?
         io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
@@ -108,11 +149,11 @@ module DerailedBenchmarks
         io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
       end
       io.puts
-      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
+      io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
       io.puts "  #{change_direction} by:"
       io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
       io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
-      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
+      io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"
@@ -122,6 +163,9 @@ module DerailedBenchmarks
       io.puts "Is significant? (max > critical): #{significant?}"
       io.puts "D critical: #{d_critical}"
       io.puts "D max: #{d_max}"
+
+      histogram(io)
+
       io.puts
     end
   end

data/lib/derailed_benchmarks/tasks.rb

@@ -1,6 +1,12 @@
 require_relative 'load_tasks'
 
 namespace :perf do
+  desc "runs the performance test against two most recent commits of the current app"
+  task :app do
+    ENV["DERAILED_PATH_TO_LIBRARY"] = '.'
+    Rake::Task["perf:library"].invoke
+  end
+
   desc "runs the same test against two different branches for statistical comparison"
   task :library do
     begin
@@ -11,94 +17,57 @@ namespace :perf do
       script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
 
       if ENV["DERAILED_PATH_TO_LIBRARY"]
-        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
+        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"].chomp
       else
         library_dir = DerailedBenchmarks.rails_path_on_disk
       end
-
-      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
-
-      # Use either the explicit SHAs when present or grab last two SHAs from commit history
-      # if only one SHA is given, then use it and the last SHA from commit history
-      branch_names = []
-      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
-      if branch_names.length < 2
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
-
-          branches = run!('git log --format="%H" -n 2').chomp.split($/)
-          if branch_names.empty?
-            branch_names = branches
-          else
-            branches.shift
-            branch_names << branches.shift
-          end
-        end
-      end
-
-      current_library_branch = ""
-      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
+      library_dir = Pathname.new(library_dir)
 
       out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
       out_dir.mkpath
 
-      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
-      branch_info = {}
-      branch_to_sha = {}
+      ref_string = ENV["SHAS_TO_TEST"] || ENV["REFS_TO_TEST"] || ""
 
-      branches_to_test.each do |branch, file|
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch}'")
-          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
-          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
-          short_sha   = run!("git rev-parse --short HEAD").strip
-          branch_to_sha[branch] = short_sha
+      project = DerailedBenchmarks::Git::SwitchProject.new(
+        path: library_dir,
+        ref_array: ref_string.split(","),
+        log_dir: out_dir
+      )
 
-          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
-        end
-        run!("#{script}")
-      end
+      stats = DerailedBenchmarks::StatsFromDir.new(project.commits)
 
+      # Advertise branch names early to make sure people know what they're testing
       puts
       puts
-      branches_to_test.each.with_index do |(branch, _), i|
-        short_sha = branch_to_sha[branch]
-        desc      = branch_info[short_sha][:desc]
-        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
+      project.commits.each_with_index do |commit, i|
+        puts "Testing #{i + 1}: #{commit.short_sha}: #{commit.description}"
       end
       puts
       puts
 
-      raise "SHAs to test must be different" if branch_info.length == 1
-      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-      puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]
+      project.restore_branch_on_return do
+        DERAILED_SCRIPT_COUNT.times do |i|
+          puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+          project.commits.each do |commit|
+            commit.checkout!
 
-      DERAILED_SCRIPT_COUNT.times do |i|
-        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
-        branches_to_test.each do |branch, file|
-          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
-          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
-        end
+            output = run!("#{script} 2>&1")
+            commit.log.open("a") {|f| f.puts output.lines.last }
+          end
 
-        if (i % 50).zero?
-          puts "Intermediate result"
-          stats.call.banner
-          puts "Continuing execution"
+          if (i % 50).zero?
+            puts "Intermediate result"
+            stats.call.banner
+            puts "Continuing execution"
+          end
         end
       end
 
     ensure
-      if library_dir && current_library_branch
-        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{current_library_branch}'")
-        end
-      end
-
       if stats
         stats.call.banner
 
-        result_file = out_dir + "results.txt"
+        result_file = out_dir.join("results.txt")
         File.open(result_file, "w") do |f|
           stats.banner(f)
         end