derailed_benchmarks 1.5.0 → 2.0.0

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.5.0"
+  VERSION = "2.0.0"
 end

data/test/derailed_benchmarks/core_ext/kernel_require_test.rb

@@ -3,7 +3,6 @@
 require 'test_helper'
 
 class KernelRequireTest < ActiveSupport::TestCase
-
   setup do
     require 'derailed_benchmarks/core_ext/kernel_require'
     GC.disable
@@ -11,23 +10,83 @@ class KernelRequireTest < ActiveSupport::TestCase
 
   teardown do
     GC.enable
+    DerailedBenchmarks::RequireTree.reset!
+  end
+
+  test "profiles load" do
+    in_fork do
+      require fixtures_dir("require/load_parent.rb")
+
+      parent = assert_node_in_parent("load_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("load_child.rb", parent)
+    end
+  end
+
+  test "profiles autoload" do
+    skip if RUBY_VERSION.start_with?("2.2") # Fails on CI, I can't install Ruby 2.2 locally to debug https://stackoverflow.com/questions/63926460/install-ruby-2-2-on-mac-osx-catalina-with-ruby-install, https://github.com/postmodern/ruby-install/issues/375
+
+    in_fork do
+      require fixtures_dir("require/autoload_parent.rb")
+      parent = assert_node_in_parent("autoload_parent.rb", TOP_REQUIRE)
+
+      assert_node_in_parent("autoload_child.rb", parent)
+    end
   end
 
+  test "core extension profiles useage" do
+    in_fork do
+      require fixtures_dir("require/parent_one.rb")
+      parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
+      assert_node_in_parent("child_one.rb", parent)
+      child_two = assert_node_in_parent("child_two.rb", parent)
+      assert_node_in_parent("relative_child", parent)
+      assert_node_in_parent("relative_child_two", parent)
+      assert_node_in_parent("raise_child.rb", child_two)
+    end
+  end
+
+  # Checks to see that the given file name is present in the
+  # parent tree node and that the memory of that file
+  # is less than the parent (since the parent should include itself
+  # plus its children)
+  #
+  # Returns the child node
   def assert_node_in_parent(file_name, parent)
     file = fixtures_dir(File.join("require", file_name))
     node = parent[file]
-    assert node,                    "Expected:\n#{parent.children}\nto include:\n#{file.inspect}"
-    assert node.cost < parent.cost, "Expected:\n#{node.inspect}\nto cost less than:\n#{parent.inspect}" unless parent == TOP_REQUIRE
+    assert node, "Expected: #{parent.name} to include: #{file.to_s} but it did not.\nChildren: #{parent.children.map(&:name).map(&:to_s)}"
+    unless parent == TOP_REQUIRE
+      assert node.cost < parent.cost, "Expected: #{node.name.inspect} (#{node.cost}) to cost less than: #{parent.name.inspect} (#{parent.cost})"
+    end
     node
   end
 
-  test "core extension profiles useage" do
-    require fixtures_dir("require/parent_one.rb")
-    parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
-    assert_node_in_parent("child_one.rb", parent)
-    child_two = assert_node_in_parent("child_two.rb", parent)
-    assert_node_in_parent("relative_child", parent)
-    assert_node_in_parent("relative_child_two", parent)
-    assert_node_in_parent("raise_child.rb", child_two)
+  # Used to get semi-clean process memory
+  # It would be better to run the requires in a totally different process
+  # but...that would take engineering
+  #
+  # If I was going to do that, I would find a way to serialize RequireTree
+  # into a json structure with file names and costs, run the script
+  # dump the json to a file, then in this process read the file and
+  # run assertions
+  def in_fork
+    Tempfile.create("stdout") do |tmp_file|
+      pid = fork do
+        $stdout.reopen(tmp_file, "w")
+        $stderr.reopen(tmp_file, "w")
+        $stdout.sync = true
+        $stderr.sync = true
+        yield
+        Kernel.exit!(0) # needed for https://github.com/seattlerb/minitest/pull/683
+      end
+      Process.waitpid(pid)
+
+      if $?.success?
+        print File.read(tmp_file)
+      else
+        raise File.read(tmp_file)
+      end
+    end
   end
 end

data/test/derailed_benchmarks/git_switch_project_test.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class GitSwitchProjectTest < ActiveSupport::TestCase
+  test "tells me when it's not pointing at a git project" do
+    exception = assert_raises {
+      DerailedBenchmarks::Git::SwitchProject.new(path: "/dev/null")
+    }
+    assert_includes(exception.message, '.git directory')
+  end
+
+  test "dirty gemspec cleaning" do
+    Dir.mktmpdir do |dir|
+      run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
+      run!("cd #{dir} && echo lol > foo.gemspec && git add .")
+
+      io = StringIO.new
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, io: io)
+
+      assert project.dirty?
+      refute project.clean?
+
+      project.restore_branch_on_return do
+        project.commits.map(&:checkout!)
+      end
+
+      assert_includes io.string, "Bundler modifies gemspec files"
+      assert_includes io.string, "Applying stash"
+    end
+  end
+
+  test "works on a git repo" do
+    Dir.mktmpdir do |dir|
+      run!("git clone https://github.com/sharpstone/default_ruby #{dir} 2>&1 && cd #{dir} && git checkout 6e642963acec0ff64af51bd6fba8db3c4176ed6e 2>&1 && git checkout -b mybranch 2>&1")
+
+      # finds shas when none given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir)
+
+      assert_equal ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "da748a59340be8b950e7bbbfb32077eb67d70c3c"], project.commits.map(&:ref)
+      first_commit = project.commits.first
+
+      assert_equal "CI test support", first_commit.description
+      assert_equal "6e64296", first_commit.short_sha
+      assert_equal "/dev/null/6e642963acec0ff64af51bd6fba8db3c4176ed6e.bench.txt", first_commit.log.to_s
+      assert_equal DateTime.parse("Tue, 14 Apr 2020 13:26:03 -0500"), first_commit.time
+
+      assert_equal "mybranch", project.current_branch_or_sha
+
+      # Finds shas when 1 is given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c"])
+
+      assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "5c09f748957d2098182762004adee27d1ff83160"], project.commits.map(&:ref)
+
+
+      # Returns correct refs if given
+      project = DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"])
+
+      assert_equal ["da748a59340be8b950e7bbbfb32077eb67d70c3c", "9b19275a592f148e2a53b87ead4ccd8c747539c9"], project.commits.map(&:ref)
+
+      first_commit = project.commits.first
+
+      first_commit.checkout!
+
+      assert_equal first_commit.short_sha, project.current_branch_or_sha
+
+      # Test restore_branch_on_return
+      project.restore_branch_on_return(quiet: true) do
+        project.commits.last.checkout!
+
+        assert_equal project.commits.last.short_sha, project.current_branch_or_sha
+      end
+
+      assert_equal project.commits.first.short_sha, project.current_branch_or_sha
+
+      exception = assert_raise {
+        DerailedBenchmarks::Git::SwitchProject.new(path: dir, ref_array: ["6e642963acec0ff64af51bd6fba8db3c4176ed6e", "mybranch"])
+      }
+
+      assert_includes(exception.message, 'Duplicate SHA resolved "6e64296"')
+    end
+  end
+end

data/test/derailed_benchmarks/require_tree_test.rb

@@ -9,7 +9,7 @@ class RequireTree < ActiveSupport::TestCase
   end
 
   def teardown
-    DerailedBenchmarks::RequireTree.const_set("REQUIRED_BY",  {})
+    DerailedBenchmarks::RequireTree.reset!
   end
 
   test "default_cost" do

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -3,6 +3,19 @@
 require 'test_helper'
 
 class StatsFromDirTest < ActiveSupport::TestCase
+  test "empty files" do
+    Dir.mktmpdir do |dir|
+      dir = Pathname.new(dir)
+      branch_info = {}
+      branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
+      branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
+      stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+      io = StringIO.new
+      stats.call.banner(io: io) # Doesn't error
+      assert_equal "", io.read
+    end
+  end
+
   test "that it works" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
@@ -29,6 +42,23 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_equal "11.3844", format % newest.median
   end
 
+  test "histogram output" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), short_sha: "5594a2d" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), short_sha: "f1ab117" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+
+    io = StringIO.new
+    stats.call.banner(io)
+
+    puts io.string
+
+    assert_match(/11\.2 , 11\.28/, io.string)
+    assert_match(/11\.8 , 11\.88/, io.string)
+  end
+
+
   test "alignment" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
@@ -48,10 +78,13 @@ class StatsFromDirTest < ActiveSupport::TestCase
 
   test "banner faster" do
     dir = fixtures_dir("stats/significant")
-    branch_info = {}
-    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
-    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
-    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    Branch_info = {}
+
+    require 'ostruct'
+    commits = []
+    commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
+    commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
+    stats = DerailedBenchmarks::StatsFromDir.new(commits).call
     newest = stats.newest
     oldest = stats.oldest
 
@@ -72,12 +105,12 @@ class StatsFromDirTest < ActiveSupport::TestCase
       11.0
     end
 
-    expected = <<~EOM
-[winner] "I am the new commit" - (10.5 seconds)
+    expected = <<-EOM
+[bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
   FASTER 🚀🚀🚀 by:
     1.0476x [older/newer]
     4.5455% [(older - newer) / older * 100]
-[loser] "Old commit" - (11.0 seconds)
+[aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
 EOM
 
     actual = StringIO.new
@@ -103,12 +136,12 @@ EOM
       11.0
     end
 
-    expected = <<~EOM
-[loser] "I am the new commit" - (11.0 seconds)
+    expected = <<-EOM
+[loser] (11.0000 seconds) "I am the new commit" ref: "loser"
   SLOWER 🐢🐢🐢 by:
      0.9545x [older/newer]
     -4.7619% [(older - newer) / older * 100]
-[winner] "Old commit" - (10.5 seconds)
+[winner] (10.5000 seconds) "Old commit" ref: "winner"
 EOM
 
     actual = StringIO.new

data/test/fixtures/require/autoload_child.rb

@@ -0,0 +1,5 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+module AutoLoadChild
+end

data/test/fixtures/require/autoload_parent.rb

@@ -0,0 +1,8 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+autoload :AutoLoadChild, File.join(__dir__, 'autoload_child.rb')
+
+if AutoLoadChild
+  # yay
+end

data/test/fixtures/require/child_one.rb

@@ -1,4 +1,4 @@
 class ChildOne
-  @retained = +""
+  @retained = String.new("")
   50_000.times.map { @retained << "A" }
 end

data/test/fixtures/require/child_two.rb

@@ -1,5 +1,5 @@
 class ChildTwo
-  @retained = +""
+  @retained = String.new("")
   200_000.times.map { @retained << "A" }
 end
 

data/test/fixtures/require/load_child.rb

@@ -0,0 +1,3 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+

data/test/fixtures/require/load_parent.rb

@@ -0,0 +1,5 @@
+@retained = String.new("")
+1_000_000.times.map { @retained << "A" }
+
+load File.join(__dir__, "load_child.rb")
+

data/test/fixtures/require/parent_one.rb

@@ -1,5 +1,5 @@
 class ParentOne
-  @retained = +""
+  @retained = String.new("")
   1_000_000.times.map { @retained << "A" }
 end
 require File.expand_path('../child_one.rb', __FILE__)

data/test/integration/tasks_test.rb

@@ -13,27 +13,60 @@ class TasksTest < ActiveSupport::TestCase
     FileUtils.remove_entry_secure(rails_app_path('tmp'))
   end
 
+  def run!(cmd)
+    puts "Running: #{cmd}"
+    out = `#{cmd}`
+    raise "Could not run #{cmd}, output: #{out}" unless $?.success?
+    out
+  end
+
   def rake(cmd, options = {})
     assert_success = options.key?(:assert_success) ? options[:assert_success] : true
     env             = options[:env]           || {}
     env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
     cmd        = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
     puts "Running: #{cmd}"
-    result = `cd '#{rails_app_path}' && #{cmd}`
-    if assert_success
-      assert $?.success?, "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
+    result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
+    if assert_success && !$?.success?
+      puts result
+      raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
     end
 
     result
   end
 
-  test 'library_branches' do
+  test 'non-rails library with branch specified' do
+    skip unless ENV['USING_RAILS_WICKED_BRANCH']
+
+    gem_path = run!("bundle info wicked --path")
+    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
+    puts rake "perf:library", { env: env }
+  end
+
+  test 'rails perf:library from git' do
+    # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
+
     skip unless ENV['USING_RAILS_GIT']
 
-    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+            "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
     puts rake "perf:library", { env: env }
   end
 
+  test "rails perf:library with bad script" do
+    # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
+
+    skip unless ENV['USING_RAILS_GIT']
+
+    error = assert_raises {
+      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2,
+              "SHAS_TO_TEST" => "acb6631cd99cdfe7db356773ef74cad7cbb570ed,12bb9d32f56883914abcd98fd72e3c68c444808d"}
+      puts rake "perf:library", { env: env }
+    }
+
+    assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
+  end
+
   test 'hitting authenticated devise apps' do
     env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
     result = rake 'perf:test', env: env
@@ -55,6 +88,13 @@ class TasksTest < ActiveSupport::TestCase
     rake "perf:test"
   end
 
+  test 'app' do
+    skip unless ENV['USING_RAILS_GIT']
+    run!("cd #{rails_app_path} && git init . && git add . && git commit -m first && git commit --allow-empty -m second")
+    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2 }
+    puts rake "perf:app", { env: env }
+  end
+
   test 'TEST_COUNT' do
     result = rake "perf:test", env: { "TEST_COUNT" => 1 }
     assert_match "1 derailed requests", result

data/test/rails_app/config/application.rb

@@ -13,6 +13,8 @@ require 'devise'
 
 module Dummy
   class Application < Rails::Application
+    config.load_defaults Rails.version.to_f
+
     config.action_mailer.default_url_options = { host: 'localhost:3000' }
 
     # Settings in config/environments/* take precedence over those specified here.

data/test/test_helper.rb

@@ -51,7 +51,6 @@ class ActiveSupport::IntegrationCase
   end
 end
 
-
 def fixtures_dir(name = "")
   root_path("test/fixtures").join(name)
 end
@@ -63,3 +62,9 @@ end
 def rails_app_path(name = "")
   root_path("test/rails_app").join(name)
 end
+
+def run!(cmd)
+  output = `#{cmd}`
+  raise "Cmd #{cmd} failed:\n#{output}" unless $?.success?
+  output
+end