derailed_benchmarks 1.4.2 → 1.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ed7fac91722a804da9125b03501653c1e1d78993b9c7d874e5e597656cb3c12
-  data.tar.gz: 5c36d946b6e0533d195f9b285d1d6541e4890a2f2eb37f2ca5d26b4c9434b655
+  metadata.gz: 348a836be042f2ebf7785075bc3f552c60ef9cefeb979d5dded1fcac5101111a
+  data.tar.gz: 6c25b2275a5c57ae9abfc6b6c55021869aa8cea9cfcfdda7422df876caf5b10d
 SHA512:
-  metadata.gz: be591fafddeff784a41a817c51f1f0f4c6a7f0b14b80f16b4792c61564af97f22a38f8cc999d5e7431cae3393cbfe427eebb291e2bc2efac7c578a802a870ec7
-  data.tar.gz: 4e0fceb32c0acbc09f74c91959121a9033b5ac188700eb225c7d51a4cee796665640302e5e7eed5508c870c63fc26a4370ed1ed6e88b15757fc8c291a70a3c88
+  metadata.gz: eb65bb19c23d4c112ae31289c3b610aca98f1541e7a4ee1a6fd1f1b98d1ace4849f061ebb323fc38b2f01ceae64907edd247e36becd849c52fa6998370faef5b
+  data.tar.gz: 0273f740b7458e1feaa59f05681e815c41ad45830fb8b5590b6860769e4f4e96ee556b5249bb90598b96e987a68bf7f3939fba9837a8573b16f0f4ce7dbc182a

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## master (unreleased)
 
+## 1.4.3
+
+- perf:library now uses median instead of average (https://github.com/schneems/derailed_benchmarks/pull/160)
+
 ## 1.4.2
 
 - Fixed syntax error that resulted in ensure end error inside tasks.rb for older rubies (https://github.com/schneems/derailed_benchmarks/pull/155)

data/README.md

@@ -431,7 +431,7 @@ Use a comma to seperate your branch names with the `SHAS_TO_TEST` env var, or om
 
 If you only include one SHA, then derailed will grab the latest commit and compare it to that SHA.
 
-These tests might take a along time to run so the output is stored on disk incase you want to see them in the future, they're at `tmp/library_branches/<timestamp>` and labeled with the same names as your commits.
+These tests might take a along time to run so the output is stored on disk incase you want to see them in the future, they're at `tmp/compare_branches/<timestamp>` and labeled with the same names as your commits.
 
 When the test is done it will output which commit "won" and by how much:
 

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -66,15 +66,27 @@ module DerailedBenchmarks
     end
 
     def x_faster
-      FORMAT % (oldest.average/newest.average).to_f
+      (oldest.median/newest.median).to_f
+    end
+
+    def faster?
+      newest.median < oldest.median
     end
 
     def percent_faster
-      FORMAT % (((oldest.average - newest.average) / oldest.average).to_f  * 100)
+      (((oldest.median - newest.median) / oldest.median).to_f  * 100)
     end
 
     def change_direction
-      newest.average < oldest.average ? "FASTER" : "SLOWER"
+      if faster?
+        "FASTER 🚀🚀🚀"
+      else
+        "SLOWER 🐢🐢🐢"
+      end
+    end
+
+    def align
+      " " * (("%i" % percent_faster).length - ("%i" % x_faster).length)
     end
 
     def banner(io = Kernel)
@@ -85,11 +97,11 @@ module DerailedBenchmarks
         io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
       end
       io.puts
-      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.average} seconds)"
+      io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
       io.puts "  #{change_direction} by:"
-      io.puts "    #{x_faster}x [older/newer]"
-      io.puts "    #{percent_faster}\% [(older - newer) / older * 100]"
-      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.average} seconds)"
+      io.puts "    #{align}#{FORMAT % x_faster}x [older/newer]"
+      io.puts "    #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
+      io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"

data/lib/derailed_benchmarks/stats_in_file.rb

@@ -30,9 +30,14 @@ module DerailedBenchmarks
     def call
       load_file!
 
+      @median = (values[(values.length - 1) / 2] + values[values.length/ 2]) / 2.0
       @average = values.inject(:+) / values.length
     end
 
+    def median
+      @median.to_f
+    end
+
     def average
       @average.to_f
     end
@@ -47,6 +52,8 @@ module DerailedBenchmarks
           raise e, "Problem with file #{@file.inspect}:\n#{@file.read}\n#{e.message}"
         end
       end
+
+      values.sort!
       values.freeze
     end
   end

data/lib/derailed_benchmarks/tasks.rb

@@ -39,7 +39,7 @@ namespace :perf do
       current_library_branch = ""
       Dir.chdir(library_dir) { current_library_branch = run!('git describe --contains --all HEAD').chomp }
 
-      out_dir = Pathname.new("tmp/library_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
+      out_dir = Pathname.new("tmp/compare_branches/#{Time.now.strftime('%Y-%m-%d-%H-%M-%s-%N')}")
       out_dir.mkpath
 
       branches_to_test = branch_names.each_with_object({}) {|elem, hash| hash[elem] = out_dir + "#{elem.gsub('/', ':')}.bench.txt" }
@@ -93,10 +93,18 @@ namespace :perf do
         end
       end
 
-      stats.call.banner if stats
+      if stats
+        stats.call.banner
+
+        result_file = out_dir + "results.txt"
+        File.open(result_file, "w") do |f|
+          stats.banner(f)
+        end
+
+        puts "Output: #{result_file.to_s}"
+      end
     end
-   end
-  
+  end
 
   desc "hits the url TEST_COUNT times"
   task :test => [:setup] do

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.4.2"
+  VERSION = "1.4.3"
 end

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -22,8 +22,11 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_in_delta 0.1730818382602285, stats.d_critical, 0.00001
     assert_equal true, stats.significant?
 
-    assert_equal "1.0062", stats.x_faster
-    assert_equal "0.6131", stats.percent_faster
+    format = DerailedBenchmarks::StatsFromDir::FORMAT
+    assert_equal "1.0062", format % stats.x_faster
+    assert_equal "0.6147", format % stats.percent_faster
+
+    assert_equal "11.3844", format % newest.median
  end
 
   test "banner faster" do
@@ -44,17 +47,17 @@ class StatsFromDirTest < ActiveSupport::TestCase
       "0.001"
     end
 
-    def newest.average
+    def newest.median
       10.5
     end
 
-    def oldest.average
+    def oldest.median
       11.0
     end
 
-    expected = <<-EOM
+    expected = <<~EOM
 [winner] "I am the new commit" - (10.5 seconds)
-  FASTER by:
+  FASTER 🚀🚀🚀 by:
     1.0476x [older/newer]
     4.5455% [(older - newer) / older * 100]
 [loser] "Old commit" - (11.0 seconds)
@@ -75,18 +78,18 @@ EOM
     newest = stats.newest
     oldest = stats.oldest
 
-    def oldest.average
+    def oldest.median
       10.5
     end
 
-    def newest.average
+    def newest.median
       11.0
     end
 
-    expected = <<-EOM
+    expected = <<~EOM
 [loser] "I am the new commit" - (11.0 seconds)
-  SLOWER by:
-    0.9545x [older/newer]
+  SLOWER 🐢🐢🐢 by:
+     0.9545x [older/newer]
     -4.7619% [(older - newer) / older * 100]
 [winner] "Old commit" - (10.5 seconds)
 EOM

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.4.3
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-11 00:00:00.000000000 Z
+date: 2019-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -257,6 +257,7 @@ files:
 - test/fixtures/stats/significant/winner.bench.txt
 - test/integration/tasks_test.rb
 - test/rails_app/Rakefile
+- test/rails_app/app/assets/config/manifest.js
 - test/rails_app/app/assets/javascripts/authenticated.js
 - test/rails_app/app/assets/stylesheets/authenticated.css
 - test/rails_app/app/controllers/application_controller.rb
@@ -341,6 +342,7 @@ test_files:
 - test/fixtures/stats/significant/winner.bench.txt
 - test/integration/tasks_test.rb
 - test/rails_app/Rakefile
+- test/rails_app/app/assets/config/manifest.js
 - test/rails_app/app/assets/javascripts/authenticated.js
 - test/rails_app/app/assets/stylesheets/authenticated.css
 - test/rails_app/app/controllers/application_controller.rb