derailed_benchmarks 1.4.2 → 1.8.0

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -3,6 +3,19 @@
 require 'test_helper'
 
 class StatsFromDirTest < ActiveSupport::TestCase
+  test "empty files" do
+    Dir.mktmpdir do |dir|
+      dir = Pathname.new(dir)
+      branch_info = {}
+      branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser"), name: "loser" }
+      branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner"), name: "winner" }
+      stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+      io = StringIO.new
+      stats.call.banner(io: io) # Doesn't error
+      assert_equal "", io.read
+    end
+  end
+
   test "that it works" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
@@ -19,19 +32,57 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_equal "loser", oldest.name
 
     assert_in_delta 0.26, stats.d_max, 0.01
-    assert_in_delta 0.1730818382602285, stats.d_critical, 0.00001
+    assert_in_delta 0.2145966026289347, stats.d_critical, 0.00001
     assert_equal true, stats.significant?
 
-    assert_equal "1.0062", stats.x_faster
-    assert_equal "0.6131", stats.percent_faster
- end
+    format = DerailedBenchmarks::StatsFromDir::FORMAT
+    assert_equal "1.0062", format % stats.x_faster
+    assert_equal "0.6147", format % stats.percent_faster
 
-  test "banner faster" do
+    assert_equal "11.3844", format % newest.median
+  end
+
+  test "histogram output" do
     dir = fixtures_dir("stats/significant")
     branch_info = {}
     branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
     branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
     stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+
+    io = StringIO.new
+    stats.call.banner(io)
+
+    assert_match(/11\.2 , 11\.28/, io.string)
+    assert_match(/11\.8 , 11\.88/, io.string)
+  end
+
+
+  test "alignment" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    def stats.percent_faster
+      -0.1
+    end
+
+    def stats.x_faster
+      0.9922
+    end
+
+    assert_equal 1, stats.align.length
+  end
+
+  test "banner faster" do
+    dir = fixtures_dir("stats/significant")
+    Branch_info = {}
+
+    require 'ostruct'
+    commits = []
+    commits << OpenStruct.new({ desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), ref: "loser", short_sha: "aaaaa" })
+    commits << OpenStruct.new({ desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), ref: "winner", short_sha: "bbbbb" })
+    stats = DerailedBenchmarks::StatsFromDir.new(commits).call
     newest = stats.newest
     oldest = stats.oldest
 
@@ -44,20 +95,20 @@ class StatsFromDirTest < ActiveSupport::TestCase
       "0.001"
     end
 
-    def newest.average
+    def newest.median
       10.5
     end
 
-    def oldest.average
+    def oldest.median
       11.0
     end
 
     expected = <<-EOM
-[winner] "I am the new commit" - (10.5 seconds)
-  FASTER by:
+[bbbbb] (10.5000 seconds) "I am the new commit" ref: "winner"
+  FASTER 🚀🚀🚀 by:
     1.0476x [older/newer]
     4.5455% [(older - newer) / older * 100]
-[loser] "Old commit" - (11.0 seconds)
+[aaaaa] (11.0000 seconds) "Old commit" ref: "loser"
 EOM
 
     actual = StringIO.new
@@ -75,20 +126,20 @@ EOM
     newest = stats.newest
     oldest = stats.oldest
 
-    def oldest.average
+    def oldest.median
       10.5
     end
 
-    def newest.average
+    def newest.median
       11.0
     end
 
     expected = <<-EOM
-[loser] "I am the new commit" - (11.0 seconds)
-  SLOWER by:
-    0.9545x [older/newer]
+[loser] (11.0000 seconds) "I am the new commit" ref: "loser"
+  SLOWER 🐢🐢🐢 by:
+     0.9545x [older/newer]
     -4.7619% [(older - newer) / older * 100]
-[winner] "Old commit" - (10.5 seconds)
+[winner] (10.5000 seconds) "Old commit" ref: "winner"
 EOM
 
     actual = StringIO.new

data/test/fixtures/require/child_one.rb

@@ -1,4 +1,4 @@
 class ChildOne
-  @retained = +""
+  @retained = String.new("")
   50_000.times.map { @retained << "A" }
 end

data/test/fixtures/require/child_two.rb

@@ -1,5 +1,5 @@
 class ChildTwo
-  @retained = +""
+  @retained = String.new("")
   200_000.times.map { @retained << "A" }
 end
 

data/test/fixtures/require/parent_one.rb

@@ -1,5 +1,5 @@
 class ParentOne
-  @retained = +""
+  @retained = String.new("")
   1_000_000.times.map { @retained << "A" }
 end
 require File.expand_path('../child_one.rb', __FILE__)

data/test/integration/tasks_test.rb

@@ -13,27 +13,58 @@ class TasksTest < ActiveSupport::TestCase
     FileUtils.remove_entry_secure(rails_app_path('tmp'))
   end
 
+  def run!(cmd)
+    puts "Running: #{cmd}"
+    out = `#{cmd}`
+    raise "Could not run #{cmd}, output: #{out}" unless $?.success?
+    out
+  end
+
   def rake(cmd, options = {})
     assert_success = options.key?(:assert_success) ? options[:assert_success] : true
     env             = options[:env]           || {}
     env_string = env.map {|key, value| "#{key.shellescape}=#{value.to_s.shellescape}" }.join(" ")
     cmd        = "env #{env_string} bundle exec rake -f perf.rake #{cmd} --trace"
     puts "Running: #{cmd}"
-    result = `cd '#{rails_app_path}' && #{cmd}`
-    if assert_success
-      assert $?.success?, "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
+    result = Bundler.with_original_env { `cd '#{rails_app_path}' && #{cmd} 2>&1` }
+    if assert_success && !$?.success?
+      puts result
+      raise "Expected '#{cmd}' to return a success status.\nOutput: #{result}"
     end
 
     result
   end
 
-  test 'library_branches' do
+  test 'non-rails library with branch specified' do
+    skip unless ENV['USING_RAILS_WICKED_BRANCH']
+
+    gem_path = run!("bundle info wicked --path")
+    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "DERAILED_PATH_TO_LIBRARY" => gem_path}
+    puts rake "perf:library", { env: env }
+  end
+
+  test 'rails perf:library from git' do
+    # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
+
     skip unless ENV['USING_RAILS_GIT']
 
-    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+    env = { "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
     puts rake "perf:library", { env: env }
   end
 
+  test "rails perf:library with bad script" do
+    # BUNDLE_GEMFILE="$(pwd)/gemfiles/rails_git.gemfile" bundle exec m test/integration/tasks_test.rb:<linenumber>
+
+    skip unless ENV['USING_RAILS_GIT']
+
+    error = assert_raises {
+      env = { "DERAILED_SCRIPT" => "nopenopenop", "TEST_COUNT" => 2, "DERAILED_SCRIPT_COUNT" => 2, "SHAS_TO_TEST" => "3054e1d584e7eca110c69a1f8423f2e0866abbf9,80f989aecece1a2b1830e9c953e5887421b52d3c"}
+      puts rake "perf:library", { env: env }
+    }
+
+    assert error.message =~ /nopenopenop:( command)? not found/, "Expected #{error.message} to include /nopenopenop: (command)? not found/ but it did not"
+  end
+
   test 'hitting authenticated devise apps' do
     env = { "PATH_TO_HIT" => "authenticated", "USE_AUTH" => "true", "TEST_COUNT" => "2" }
     result = rake 'perf:test', env: env
@@ -55,6 +86,13 @@ class TasksTest < ActiveSupport::TestCase
     rake "perf:test"
   end
 
+  test 'app' do
+    skip unless ENV['USING_RAILS_GIT']
+    run!("cd #{rails_app_path} && git init . && git add . && git commit -m first && git commit --allow-empty -m second")
+    env = { "TEST_COUNT" => 10, "DERAILED_SCRIPT_COUNT" => 2 }
+    puts rake "perf:app", { env: env }
+  end
+
   test 'TEST_COUNT' do
     result = rake "perf:test", env: { "TEST_COUNT" => 1 }
     assert_match "1 derailed requests", result

data/test/rails_app/config/application.rb

@@ -13,6 +13,8 @@ require 'devise'
 
 module Dummy
   class Application < Rails::Application
+    config.load_defaults Rails.version.to_f
+
     config.action_mailer.default_url_options = { host: 'localhost:3000' }
 
     # Settings in config/environments/* take precedence over those specified here.

data/test/test_helper.rb

@@ -63,3 +63,9 @@ end
 def rails_app_path(name = "")
   root_path("test/rails_app").join(name)
 end
+
+def run!(cmd)
+  output = `#{cmd}`
+  raise "Cmd #{cmd} failed:\n#{output}" unless $?.success?
+  output
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.8.0
 platform: ruby
 authors:
 - Richard Schneeman
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-11 00:00:00.000000000 Z
+date: 2020-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -104,16 +104,22 @@ dependencies:
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.19'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.19'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: ruby-statistics
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +134,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: mini_histogram
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
@@ -165,7 +185,7 @@ dependencies:
         version: '3'
     - - "<="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -175,7 +195,7 @@ dependencies:
         version: '3'
     - - "<="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: devise
   requirement: !ruby/object:Gem::Requirement
@@ -237,13 +257,18 @@ files:
 - lib/derailed_benchmarks/auth_helper.rb
 - lib/derailed_benchmarks/auth_helpers/devise.rb
 - lib/derailed_benchmarks/core_ext/kernel_require.rb
+- lib/derailed_benchmarks/git/commit.rb
+- lib/derailed_benchmarks/git/in_path.rb
+- lib/derailed_benchmarks/git/switch_project.rb
+- lib/derailed_benchmarks/git_switch_project.rb
 - lib/derailed_benchmarks/load_tasks.rb
 - lib/derailed_benchmarks/require_tree.rb
+- lib/derailed_benchmarks/stats_for_file.rb
 - lib/derailed_benchmarks/stats_from_dir.rb
-- lib/derailed_benchmarks/stats_in_file.rb
 - lib/derailed_benchmarks/tasks.rb
 - lib/derailed_benchmarks/version.rb
 - test/derailed_benchmarks/core_ext/kernel_require_test.rb
+- test/derailed_benchmarks/git_switch_project_test.rb
 - test/derailed_benchmarks/require_tree_test.rb
 - test/derailed_benchmarks/stats_from_dir_test.rb
 - test/derailed_test.rb
@@ -257,6 +282,7 @@ files:
 - test/fixtures/stats/significant/winner.bench.txt
 - test/integration/tasks_test.rb
 - test/rails_app/Rakefile
+- test/rails_app/app/assets/config/manifest.js
 - test/rails_app/app/assets/javascripts/authenticated.js
 - test/rails_app/app/assets/stylesheets/authenticated.css
 - test/rails_app/app/controllers/application_controller.rb
@@ -307,7 +333,7 @@ homepage: https://github.com/schneems/derailed_benchmarks
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -315,19 +341,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Benchmarks designed to performance test your ENTIRE site
 test_files:
 - test/derailed_benchmarks/core_ext/kernel_require_test.rb
+- test/derailed_benchmarks/git_switch_project_test.rb
 - test/derailed_benchmarks/require_tree_test.rb
 - test/derailed_benchmarks/stats_from_dir_test.rb
 - test/derailed_test.rb
@@ -341,6 +368,7 @@ test_files:
 - test/fixtures/stats/significant/winner.bench.txt
 - test/integration/tasks_test.rb
 - test/rails_app/Rakefile
+- test/rails_app/app/assets/config/manifest.js
 - test/rails_app/app/assets/javascripts/authenticated.js
 - test/rails_app/app/assets/stylesheets/authenticated.css
 - test/rails_app/app/controllers/application_controller.rb