derailed_benchmarks 1.4.3 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 348a836be042f2ebf7785075bc3f552c60ef9cefeb979d5dded1fcac5101111a
-  data.tar.gz: 6c25b2275a5c57ae9abfc6b6c55021869aa8cea9cfcfdda7422df876caf5b10d
+  metadata.gz: 596677a7dc15b62f384bbf140c0bbe05eb9c890b73ea66e76c34c115ba0c38c1
+  data.tar.gz: 194a74aac18253ad77bb4fcbb464a70478d8127ea894df86e3b8071d6390e264
 SHA512:
-  metadata.gz: eb65bb19c23d4c112ae31289c3b610aca98f1541e7a4ee1a6fd1f1b98d1ace4849f061ebb323fc38b2f01ceae64907edd247e36becd849c52fa6998370faef5b
-  data.tar.gz: 0273f740b7458e1feaa59f05681e815c41ad45830fb8b5590b6860769e4f4e96ee556b5249bb90598b96e987a68bf7f3939fba9837a8573b16f0f4ce7dbc182a
+  metadata.gz: 00bc97b216b94edf416d320749b789793f2c6dd2e35b7b56ddd45daaaaec88c61d972cc2b8f52005f9c6cb81f564797377dcd519639b41af366eafe8f88451be
+  data.tar.gz: b0bd2f96ca277f47f1696d7339d10bc9e0c580607def3e5bb5bd3550302749bc53a4575f353bb34c7d104e1c7be4e712af44221b7094a04b636f093b1dd0035a

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## master (unreleased)
 
+## 1.5.0
+
+- Test `perf:library` results against 99% confidence interval in addition to 95% (https://github.com/schneems/derailed_benchmarks/pull/165)
+- Change default, `perf:library` tests do not stop automatically any more (https://github.com/schneems/derailed_benchmarks/pull/164)
+
+## 1.4.4
+
+- Fix alignment of deicmals in output (https://github.com/schneems/derailed_benchmarks/pull/161)
+
 ## 1.4.3
 
 - perf:library now uses median instead of average (https://github.com/schneems/derailed_benchmarks/pull/160)

data/README.md

@@ -453,7 +453,7 @@ You can use this to test changes in other libraries that aren't rails, you just
 
 > To get the best results before running tests you should close all programs on your laptop, turn on a program to prevent your laptop from going to sleep (or increase your sleep timer). Make sure it's plugged into a power outlet and  go grab a cup of coffee. If you do anything on your laptop while this test is running you risk the chance of skewing your results.
 
-By default derailed will stop once statistical signficance has been detected, you can tune this behavior by setting `DERAILED_STOP_VALID_COUNT` env var. Setting this to a positive number, will increase the number of iterations required that are detected to be statistically significant. For example setting it to 100 might result in 120 runs if it takes 20 runs to detect significance. Generally the more runs you have, the more accurate your averages will be. You can disable this all together by setting `DERAILED_STOP_VALID_COUNT=0` which will force derailed to run all iterations.
+As the test is executing, intermediate results will be printed every 50 iterations.
 
 ## Environment Variables
 

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -42,14 +42,25 @@ module DerailedBenchmarks
 
     def call
       @files.each(&:call)
-      @stats = statistical_test
+
+      stats_95 = statistical_test(confidence: 95)
+
+      # If default check is good, see if we also pass a more rigorous test
+      # if so, then use the more rigourous test
+      if stats_95[:alternative]
+        stats_99 = statistical_test(confidence: 99)
+        @stats = stats_99 if stats_99[:alternative]
+      end
+      @stats ||= stats_95
+
       self
     end
 
-    def statistical_test(series_1=oldest.values, series_2=newest.values)
+    def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
       StatisticalTest::KSTest.two_samples(
         group_one: series_1,
-        group_two: series_2
+        group_two: series_2,
+        alpha: (100 - confidence) / 100.0
       )
     end
 
@@ -86,7 +97,7 @@ module DerailedBenchmarks
     end
 
     def align
-      " " * (("%i" % percent_faster).length - ("%i" % x_faster).length)
+      " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
     end
 
     def banner(io = Kernel)
@@ -107,6 +118,7 @@ module DerailedBenchmarks
       io.puts "Samples: #{newest.values.length}"
       io.puts
       io.puts "Test type: Kolmogorov Smirnov"
+      io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
       io.puts "Is significant? (max > critical): #{significant?}"
       io.puts "D critical: #{d_critical}"
       io.puts "D max: #{d_max}"

data/lib/derailed_benchmarks/tasks.rb

@@ -71,18 +71,20 @@ namespace :perf do
 
       raise "SHAs to test must be different" if branch_info.length == 1
       stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-      ENV["DERAILED_STOP_VALID_COUNT"] ||= "50"
-      stop_valid_count = Integer(ENV["DERAILED_STOP_VALID_COUNT"])
+      puts "Env var no longer has any affect DERAILED_STOP_VALID_COUNT" if ENV["DERAILED_STOP_VALID_COUNT"]
 
-      times_significant = 0
       DERAILED_SCRIPT_COUNT.times do |i|
         puts "Sample: #{i.next}/#{DERAILED_SCRIPT_COUNT} iterations per sample: #{ENV['TEST_COUNT']}"
         branches_to_test.each do |branch, file|
           Dir.chdir(library_dir) { run!("git checkout '#{branch}'") }
           run!(" #{script} 2>&1 | tail -n 1 >> '#{file}'")
         end
-        times_significant += 1 if i >= 2 && stats.call.significant?
-        break if stop_valid_count != 0 && times_significant == stop_valid_count
+
+        if (i % 50).zero?
+          puts "Intermediate result"
+          stats.call.banner
+          puts "Continuing execution"
+        end
       end
 
     ensure

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.4.3"
+  VERSION = "1.5.0"
 end

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -19,7 +19,7 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_equal "loser", oldest.name
 
     assert_in_delta 0.26, stats.d_max, 0.01
-    assert_in_delta 0.1730818382602285, stats.d_critical, 0.00001
+    assert_in_delta 0.2145966026289347, stats.d_critical, 0.00001
     assert_equal true, stats.significant?
 
     format = DerailedBenchmarks::StatsFromDir::FORMAT
@@ -27,7 +27,24 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_equal "0.6147", format % stats.percent_faster
 
     assert_equal "11.3844", format % newest.median
- end
+  end
+
+  test "alignment" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    def stats.percent_faster
+      -0.1
+    end
+
+    def stats.x_faster
+      0.9922
+    end
+
+    assert_equal 1, stats.align.length
+  end
 
   test "banner faster" do
     dir = fixtures_dir("stats/significant")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.0
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-17 00:00:00.000000000 Z
+date: 2019-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -323,7 +323,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Benchmarks designed to performance test your ENTIRE site