derailed_benchmarks 1.7.0 → 2.1.0

data/lib/derailed_benchmarks/git/in_path.rb

@@ -0,0 +1,59 @@
+module DerailedBenchmarks
+  # A class for running commands in a git directory
+  #
+  # It's faster to check if we're already in that directory instead
+  # of having to `cd` into each time. https://twitter.com/schneems/status/1305196730170961920
+  #
+  # Example:
+  #
+  #   in_git_path = InGitPath.new(`bundle info heapy --path`.strip)
+  #   in_git_path.checkout!("f0f92b06156f2274021aa42f15326da041ee9009")
+  #   in_git_path.short_sha # => "f0f92b0"
+  class Git::InPath
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def description
+      run!("git log --oneline --format=%B -n 1 HEAD | head -n 1")
+    end
+
+    def short_sha
+      run!("git rev-parse --short HEAD")
+    end
+
+    def time_stamp_string
+      run!("git log -n 1 --pretty=format:%ci") # https://stackoverflow.com/a/25921837/147390
+    end
+
+    def branch
+      branch = run!("git rev-parse --abbrev-ref HEAD")
+      branch == "HEAD" ? nil : branch
+    end
+
+    def checkout!(ref)
+      run!("git checkout '#{ref}' 2>&1")
+    end
+
+    def time
+      DateTime.parse(time_stamp_string)
+    end
+
+    def run(cmd)
+      if Dir.pwd == path
+        out = `#{cmd}`.strip
+      else
+        out = `cd #{path} && #{cmd}`.strip
+      end
+      out
+    end
+
+    def run!(cmd)
+      out = run(cmd)
+      raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?
+      out
+    end
+  end
+end

data/lib/derailed_benchmarks/git/switch_project.rb

@@ -0,0 +1,128 @@
+module DerailedBenchmarks
+  class Git
+  end
+end
+require_relative "in_path.rb"
+require_relative "commit.rb"
+
+module DerailedBenchmarks
+  # Wraps two or more git commits in a specific location
+  #
+  # Returns an array of GitCommit objects that can be used to manipulate
+  # and checkout the repo
+  #
+  # Example:
+  #
+  #   `git clone https://sharpstone/default_ruby tmp/default_ruby`
+  #
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby")
+  #
+  # By default it will represent the last two commits:
+  #
+  #   project.commits.length # => 2
+  #
+  # You can pass in explicit REFs in an array:
+  #
+  #   ref_array = ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"]
+  #   project = GitSwitchProject.new(path: "tmp/default_ruby", ref_array: ref_array)
+  #
+  #   puts project.commits.map(&:ref) == ref_array # => true
+  #
+  #
+  # It knows the current branch or sha:
+  #
+  #    `cd tmp/ruby && git checkout -b mybranch`
+  #    project.current_branch_or_sha #=> "mybranch"
+  #
+  # It can be used for safely wrapping checkouts to ensure the project returns to it's original branch:
+  #
+  #    project.restore_branch_on_return do
+  #      project.commits.first.checkout!
+  #      project.current_branch_or_sha # => "da748a593"
+  #    end
+  #
+  #    project.current_branch_or_sha # => "mybranch"
+  class Git::SwitchProject
+    attr_reader :commits
+
+    def initialize(path: , ref_array: [], io: STDOUT, log_dir: "/dev/null")
+      @path = Pathname.new(path)
+
+      @in_git_path = Git::InPath.new(@path.expand_path)
+
+      raise "Must be a path with a .git directory '#{@path}'" if !@path.join(".git").exist?
+      @io = io
+      @commits = []
+      log_dir = Pathname(log_dir)
+
+      expand_refs(ref_array).each do |ref|
+        restore_branch_on_return(quiet: true) do
+          @commits << Git::Commit.new(path: @path, ref: ref, log_dir: log_dir)
+        end
+      end
+
+      if (duplicate = @commits.group_by(&:short_sha).detect {|(k, v)| v.length > 1})
+        raise "Duplicate SHA resolved #{duplicate[0].inspect}: #{duplicate[1].map {|c| "'#{c.ref}' => '#{c.short_sha}'"}.join(", ") } at #{@path}"
+      end
+    end
+
+    def current_branch_or_sha
+      branch_or_sha = @in_git_path.branch
+      branch_or_sha ||= @in_git_path.short_sha
+      branch_or_sha
+    end
+
+    def dirty?
+      !clean?
+    end
+
+    # https://stackoverflow.com/a/3879077/147390
+    def clean?
+      @in_git_path.run("git diff-index --quiet HEAD --") && $?.success?
+    end
+
+    private def status(pattern: "*.gemspec")
+      @in_git_path.run("git status #{pattern}")
+    end
+
+    def restore_branch_on_return(quiet: false)
+      if dirty? && status.include?("gemspec")
+        dirty_gemspec = true
+        unless quiet
+          @io.puts "Working tree at #{@path} is dirty, stashing. This will be popped on return"
+          @io.puts "Bundler modifies gemspec files on git install, this is normal"
+          @io.puts "Original status:\n#{status}"
+        end
+        @in_git_path.run!("git stash")
+      end
+      branch_or_sha = self.current_branch_or_sha
+      yield
+    ensure
+      return unless branch_or_sha
+      @io.puts "Resetting git dir of '#{@path.to_s}' to #{branch_or_sha.inspect}" unless quiet
+
+      @in_git_path.checkout!(branch_or_sha)
+      if dirty_gemspec
+        out = @in_git_path.run!("git stash apply 2>&1")
+        @io.puts "Applying stash of '#{@path.to_s}':\n#{out}" unless quiet
+      end
+    end
+
+    # case ref_array.length
+    # when >= 2
+    #   returns original array
+    # when 1
+    #   returns the given ref plus the one before it
+    # when 0
+    #   returns the most recent 2 refs
+    private def expand_refs(ref_array)
+      return ref_array if ref_array.length >= 2
+
+      @in_git_path.checkout!(ref_array.first) if ref_array.first
+
+      branches_string = @in_git_path.run!("git log --format='%H' -n 2")
+      ref_array = branches_string.split($/)
+      return ref_array
+    end
+  end
+end

data/lib/derailed_benchmarks/git_switch_project.rb

@@ -0,0 +1 @@
+

data/lib/derailed_benchmarks/load_tasks.rb

@@ -110,7 +110,7 @@ namespace :perf do
           STDERR.puts "Bad request to #{cmd.inspect} \n\n***RESPONSE***:\n\n#{ response.inspect }"
 
           FileUtils.mkdir_p("tmp")
-          File.open("tmp/fail.html", "w+") {|f| f.write response.body }
+          File.open("tmp/fail.html", "w+") {|f| f.write response }
 
           `open #{File.expand_path("tmp/fail.html")}` if ENV["DERAILED_DEBUG"]
 

data/lib/derailed_benchmarks/require_tree.rb

@@ -13,6 +13,16 @@ module DerailedBenchmarks
     def initialize(name)
       @name     = name
       @children = {}
+      @cost = 0
+    end
+
+    def self.reset!
+      REQUIRED_BY.clear
+      if defined?(Kernel::REQUIRE_STACK)
+        Kernel::REQUIRE_STACK.clear
+
+        Kernel::REQUIRE_STACK.push(TOP_REQUIRE)
+      end
     end
 
     def <<(tree)
@@ -40,7 +50,7 @@ module DerailedBenchmarks
     end
 
     def to_string
-      str = +"#{name}: #{cost.round(4)} MiB"
+      str = String.new("#{name}: #{cost.round(4)} MiB")
       if parent && REQUIRED_BY[self.name.to_s]
         names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s]
         if names.any?

data/lib/derailed_benchmarks/{stats_in_file.rb → stats_for_file.rb}

@@ -16,24 +16,30 @@ module DerailedBenchmarks
   #  x.average # => 10.5
   #  x.name    # => "muhfile"
   class StatsForFile
-    attr_reader :name, :values, :desc, :time
+    attr_reader :name, :values, :desc, :time, :short_sha
 
-    def initialize(file:, name:, desc: "", time: )
+    def initialize(file:, name:, desc: "", time: , short_sha: nil)
       @file = Pathname.new(file)
       FileUtils.touch(@file)
 
       @name = name
       @desc = desc
       @time = time
+      @short_sha = short_sha
     end
 
     def call
       load_file!
+      return if values.empty?
 
       @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
       @average = values.inject(:+) / values.length
     end
 
+    def empty?
+      values.empty?
+    end
+
     def median
       @median.to_f
     end

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -2,9 +2,9 @@
 
 require 'bigdecimal'
 require 'statistics'
-require 'unicode_plot'
 require 'stringio'
 require 'mini_histogram'
+require 'mini_histogram/plot'
 
 module DerailedBenchmarks
   # A class used to read several benchmark files
@@ -29,14 +29,28 @@ module DerailedBenchmarks
     FORMAT = "%0.4f"
     attr_reader :stats, :oldest, :newest
 
-    def initialize(hash)
+    def initialize(input)
       @files = []
 
-      hash.each do |branch, info_hash|
-        file = info_hash.fetch(:file)
-        desc = info_hash.fetch(:desc)
-        time = info_hash.fetch(:time)
-        @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
+      if input.is_a?(Hash)
+        hash = input
+        hash.each do |branch, info_hash|
+          file = info_hash.fetch(:file)
+          desc = info_hash.fetch(:desc)
+          time = info_hash.fetch(:time)
+          short_sha = info_hash[:short_sha]
+          @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch, short_sha: short_sha)
+        end
+      else
+        input.each do |commit|
+          @files << StatsForFile.new(
+            file: commit.file,
+            desc: commit.desc,
+            time: commit.time,
+            name: commit.ref,
+            short_sha: commit.short_sha
+          )
+        end
       end
       @files.sort_by! { |f| f.time }
       @oldest = @files.first
@@ -46,6 +60,8 @@ module DerailedBenchmarks
     def call
       @files.each(&:call)
 
+      return self if @files.detect(&:empty?)
+
       stats_95 = statistical_test(confidence: 95)
 
       # If default check is good, see if we also pass a more rigorous test
@@ -104,25 +120,28 @@ module DerailedBenchmarks
     end
 
     def histogram(io = $stdout)
-      newest_histogram = MiniHistogram.new(newest.values)
-      oldest_histogram = MiniHistogram.new(oldest.values)
-      MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
-
-      {newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
-        plot = UnicodePlot.histogram(
-          histogram,
-          title: "\n#{' ' * 18 }Histogram - [#{report.name}] #{report.desc.inspect}",
-          ylabel: "Time (s)",
+      dual_histogram = MiniHistogram.dual_plot do |a, b|
+        a.values = newest.values
+        a.options = {
+          title: "\n   [#{newest.short_sha || newest.name}] description:\n     #{newest.desc.inspect}",
+          xlabel: "# of runs in range"
+        }
+        b.values = oldest.values
+        b.options = {
+          title: "\n   [#{oldest.short_sha || oldest.name}] description:\n     #{oldest.desc.inspect}",
           xlabel: "# of runs in range"
-        )
-        plot.render(io)
-        io.puts
+        }
       end
 
       io.puts
+      io.puts "Histograms (time ranges are in seconds):"
+      io.puts(dual_histogram)
+      io.puts
     end
 
     def banner(io = $stdout)
+      return if @files.detect(&:empty?)
+
       io.puts
       if significant?
         io.puts "❤️ ❤️ ❤️  (Statistically Significant) ❤️ ❤️ ❤️"
@@ -130,11 +149,11 @@ module DerailedBenchmarks
         io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
       end
       io.puts
-      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
+      io.puts "[#{newest.short_sha || newest.name}] (#{FORMAT % newest.median} seconds) #{newest.desc.inspect} ref: #{newest.name.inspect}"
       io.puts "  #{change_direction} by:"
       io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
       io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
-      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
+      io.puts "[#{oldest.short_sha || oldest.name}] (#{FORMAT % oldest.median} seconds) #{oldest.desc.inspect} ref: #{oldest.name.inspect}"
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"

data/lib/derailed_benchmarks/tasks.rb

@@ -17,96 +17,57 @@ namespace :perf do
       script = ENV["DERAILED_SCRIPT"] || "bundle exec derailed exec perf:test"
 
       if ENV["DERAILED_PATH_TO_LIBRARY"]
-        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"]
+        library_dir = ENV["DERAILED_PATH_TO_LIBRARY"].chomp
       else
         library_dir = DerailedBenchmarks.rails_path_on_disk
       end
-
-      raise "Must be a path with a .git directory '#{library_dir}'" unless File.exist?(File.join(library_dir, ".git"))
-
-      # Use either the explicit SHAs when present or grab last two SHAs from commit history
-      # if only one SHA is given, then use it and the last SHA from commit history
-      branch_names = []
-      branch_names = ENV.fetch("SHAS_TO_TEST").split(",") if ENV["SHAS_TO_TEST"]
-      if branch_names.length < 2
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch_names.first}'") unless branch_names.empty?
-
-          branches = run!('git log --format="%H" -n 2').chomp.split($/)
-          if branch_names.empty?
-            branch_names = branches
-          else
-            branches.shift
-            branch_names << branches.shift
-          end
-        end
-      end
-
-      current_library_branch = ""
-      Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
+      library_dir = Pathname.new(library_dir)
 
       out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
       out_dir.mkpath
 
-      branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
-      branch_info = {}
-      branch_to_sha = {}
+      ref_string = ENV["SHAS_TO_TEST"] || ENV["REFS_TO_TEST"] || ""
 
-      branches_to_test.each do |branch, file|
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{branch}'")
-          description = run!("git log --oneline --format=%B -n 1 HEAD | head -n 1").strip
-          time_stamp  = run!("git log -n 1 --pretty=format:%ci").strip # https://stackoverflow.com/a/25921837/147390
-          short_sha   = run!("git rev-parse --short HEAD").strip
-          branch_to_sha[branch] = short_sha
+      project = DerailedBenchmarks::Git::SwitchProject.new(
+        path: library_dir,
+        ref_array: ref_string.split(","),
+        log_dir: out_dir
+      )
 
-          branch_info[short_sha] = { desc: description, time: DateTime.parse(time_stamp), file: file }
-        end
-        run!("#{script}")
-      end
+      stats = DerailedBenchmarks::StatsFromDir.new(project.commits)
 
+      # Advertise branch names early to make sure people know what they're testing
       puts
       puts
-      branches_to_test.each.with_index do |(branch, _), i|
-        short_sha = branch_to_sha[branch]
-        desc      = branch_info[short_sha][:desc]
-        puts "Testing #{i + 1}: #{short_sha}: #{desc}"
+      project.commits.each_with_index do |commit, i|
+        puts "Testing #{i + 1}: #{commit.short_sha}: #{commit.description}"
       end
       puts
       puts
 
-      raise "SHAs to test must be different" if branch_info.length == 1
-      stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-      puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]
+      project.restore_branch_on_return do
+        DERAILED_SCRIPT_COUNT.times do |i|
+          puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
+          project.commits.each do |commit|
+            commit.checkout!
 
-      DERAILED_SCRIPT_COUNT.times do |i|
-        puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
-        branches_to_test.each do |branch, file|
-          Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
-          run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
-        end
+            output = run!("#{script} 2>&1")
+            commit.log.open("a") {|f| f.puts output.lines.last }
+          end
 
-        if (i % 50).zero?
-          puts "Intermediate result"
-          stats.call
-          stats.banner
-          puts "Continuing execution"
+          if (i % 50).zero?
+            puts "Intermediate result"
+            stats.call.banner
+            puts "Continuing execution"
+          end
         end
       end
 
     ensure
-      if library_dir && current_library_branch
-        puts "Resetting git dir of '#{library_dir.to_s}' to #{current_library_branch.inspect}"
-        Dir.chdir(library_dir) do
-          run!("git checkout '#{current_library_branch}'")
-        end
-      end
-
       if stats
-        stats.call
-        stats.banner
+        stats.call.banner
 
-        result_file = out_dir + "results.txt"
+        result_file = out_dir.join("results.txt")
         File.open(result_file, "w") do |f|
           stats.banner(f)
         end
@@ -286,6 +247,42 @@ namespace :perf do
     puts "Also try uploading #{file_name.inspect} to http://tenderlove.github.io/heap-analyzer/"
   end
 
+  desc "three heaps generation for comparison."
+  task :heap_diff => [:setup] do
+    require 'objspace'
+
+    launch_time = Time.now.iso8601
+    FileUtils.mkdir_p("tmp")
+    ObjectSpace.trace_object_allocations_start
+    3.times do |i|
+      file_name = "tmp/#{launch_time}-heap-#{i}.ndjson"
+      puts "Running #{ TEST_COUNT } times"
+      TEST_COUNT.times {
+        call_app
+      }
+      GC.start
+
+      puts "Heap file generated: #{ file_name.inspect }"
+      ObjectSpace.dump_all(output: File.open(file_name, 'w'))
+    end
+
+    require 'heapy'
+
+    puts ""
+    puts "Diff"
+    puts "===="
+    Heapy::Diff.new(
+      before: "tmp/#{launch_time}-heap-0.ndjson",
+      after: "tmp/#{launch_time}-heap-1.ndjson",
+      retained: "tmp/#{launch_time}-heap-2.ndjson"
+    ).call
+
+    puts ""
+    puts "Run `$ heapy --help` for more options"
+    puts ""
+    puts "Also read https://speakerdeck.com/samsaffron/why-ruby-2-dot-1-excites-me?slide=27 to understand better what you are reading."
+  end
+
   def run!(cmd)
     out = `#{cmd}`
     raise "Error while running #{cmd.inspect}: #{out}" unless $?.success?