benchmark-ips 2.6.1 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2edc0877b8b37420698e685b9db778b2bad7c094
-  data.tar.gz: d034c9416613ed22534e5a2e7b4da8c989313fb8
+  metadata.gz: 122a521904ed1263289b5f68e76c72515830447e
+  data.tar.gz: 6b60d87ce82746370a1d0bff1f49be8175b2a5be
 SHA512:
-  metadata.gz: f807342d963e71a638e77db1f1eeb15f1122fffc996a5e98874d7691821407000d83eba71b081fb0951ef0a6cd10f04e1c951698d0486d6b61561150c6b58631
-  data.tar.gz: a47ebd2708d984bdbf960ca50937021f5d2439a216cebf9ec23fe68178b4a6acfdae1e45588748ff182211311950f775c0ab5a4620ec093707c02a8bbf685e7c
+  metadata.gz: 3e3588122227405ccb4ce7376e431ee630ca87779633cecdf24b82ba672610cbafc92ff82685b05accda763a54a0c54b50fe2a42763c5f45a9a2171984aaec69
+  data.tar.gz: cf47674032d38ac758b371edae0b1ffeceaac07ad9f7fc10d8413ffeb2db173395cf90d187b41293d122c50f6c2a9c68776b1e15c6c0c425549928d697d11485

data/History.txt

@@ -1,3 +1,22 @@
+=== 2.7.0 / 2016-08-05
+
+* 1 minor features:
+  * Add support for confidence intervals
+
+* 1 bug fixes:
+  * Cleanup a few coding patterns
+
+* 2 doc fixes:
+  * Add infos about benchark.fyi to Readme
+  * Remove ancient releases
+
+* 3 merged PRs:
+  * Merge pull request #65 from kbrock/fixup_inject
+  * Merge pull request #67 from benoittgt/master
+  * Merge pull request #69 from chrisseaton/kalibera-confidence-intervals
+
+=== MISSING 2.6.0 and 2.6.1
+
 === 2.5.0 / 2016-02-14
 
 * 1 minor feature:

data/README.md

@@ -144,10 +144,10 @@ are independent of each other. You can do this with the `hold!` command.
 
 ```ruby
 Benchmark.ips do |x|
-  
+
   # Hold results between multiple invocations of Ruby
   x.hold! 'filename'
-  
+
 end
 ```
 
@@ -172,9 +172,58 @@ Benchmark.ips do |x|
   x.config(:iterations => 3)
 
     # or
-  
+
   x.iterations = 3
+
+end
+```
+
+### Online sharing
+
+If you want to share quickly your benchmark result with others. Run you benchmark
+with `SHARE=1` argument. I.e.: `SHARE=1 ruby my_benchmark.rb`.
+Result will be sent to [benchmark.fyi](https://benchmark.fyi/) and benchmark-ips
+will display the link to share the benchmark's result.
+
+### Advanced Statistics
+
+By default, the margin of error shown is plus-minus one standard deviation. If
+a more advanced statistical test is wanted, a bootstrap confidence interval
+can be calculated instead. A bootstrap confidence interval has the advantages of
+arguably being more mathematically sound for this application than a standard
+deviation, it additionally produces an error for relative slowdowns, which the
+standard deviation does not, and it is arguably more intuitive and actionable.
+
+When a bootstrap confidence interval is used, a median of the interval is used
+rather than the mean of the samples, which is what you get with the default
+standard deviation.
+
+The bootstrap confidence interval used is the one described by Tomas Kalibera.
+Note that for this technique to be valid your benchmark should have reached a
+non-periodic steady state with statistically independent samples (it should
+have warmed up) by the time measurements start.
+
+Using a bootstrap confidence internal requires that the 'kalibera' gem is
+installed separately. This gem is not a formal dependency, as by default it is
+not needed.
+
+```
+gem install kalibera
+```
+
+```ruby
+Benchmark.ips do |x|
+
+  # The default is :stats => :sd, which doesn't have a configurable confidence
+  x.config(:stats => :bootstrap, :confidence => 95)
+
+    # or
+
+  x.stats = :bootstrap
+  x.confidence = 95
   
+  # confidence is 95% by default, so it can be omitted
+
 end
 ```
 

data/lib/benchmark/compare.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Benchmark
   # Functionality of performaing comparison between reports.
   #
@@ -32,33 +34,40 @@ module Benchmark
     def compare(*entries)
       return if entries.size < 2
 
-      sorted = entries.sort_by(&:ips).reverse
+      sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
 
       best = sorted.shift
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.ips
+      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
         
-        $stdout.printf "%20s: %10.1f i/s - ", name, report.ips
+        $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
         
-        best_low = best.ips - best.ips_sd
-        report_high = report.ips + report.ips_sd
+        best_low = best.stats.central_tendency - best.stats.error
+        report_high = report.stats.central_tendency + report.stats.error
         overlaps = report_high > best_low 
         
         if overlaps
           $stdout.print "same-ish: difference falls within error"
         else
-          x = (best.ips.to_f / report.ips.to_f)
-          $stdout.printf "%.2fx slower", x
+          slowdown, error = report.stats.slowdown(best.stats)
+          $stdout.printf "%.2fx ", slowdown
+          if error
+            $stdout.printf " (± %.2f)", error
+          end
+          $stdout.print " slower"
         end
         
         $stdout.puts
       end
 
+      footer = best.stats.footer
+      $stdout.puts footer.rjust(40) if footer
+
       $stdout.puts
     end
   end

data/lib/benchmark/ips.rb

@@ -1,6 +1,8 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/sd'
+require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
 require 'benchmark/ips/job/entry'
 require 'benchmark/ips/job/stdout_report'
@@ -13,10 +15,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.6.1"
+    VERSION = "2.7.0"
 
     # CODENAME of current version.
-    CODENAME = "Sharing is Caring"
+    CODENAME = "Cultivating Confidence"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.

data/lib/benchmark/ips/job.rb

@@ -42,6 +42,14 @@ module Benchmark
       # @return [Integer]
       attr_accessor :iterations
 
+      # Statistics model.
+      # @return [Object]
+      attr_accessor :stats
+
+      # Confidence.
+      # @return [Integer]
+      attr_accessor :confidence
+
       # Instantiate the Benchmark::IPS::Job.
       # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
       # @option opts [Boolean] (false) :quiet Suppress the printing of information.
@@ -61,6 +69,10 @@ module Benchmark
         @warmup = 2
         @time = 5
         @iterations = 1
+
+        # Default statistical model
+        @stats = :sd
+        @confidence = 95
       end
 
       # Job configuration options, set +@warmup+ and +@time+.
@@ -72,6 +84,8 @@ module Benchmark
         @time = opts[:time] if opts[:time]
         @suite = opts[:suite] if opts[:suite]
         @iterations = opts[:iterations] if opts[:iterations]
+        @stats = opts[:stats] if opts[:stats]
+        @confidence = opts[:confidence] if opts[:confidence]
       end
 
       # Return true if job needs to be compared.
@@ -133,8 +147,7 @@ module Benchmark
       # @return [Integer] Cycles per 100ms.
       def cycles_per_100ms time_msec, iters
         cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
-        cycles = 1 if cycles <= 0
-        cycles
+        cycles <= 0 ? 1 : cycles
       end
 
       # Calculate the time difference of before and after in microseconds.
@@ -179,6 +192,8 @@ module Benchmark
         @iterations.times do |n|
           held = run_benchmark
         end
+
+        @stdout.footer if @stdout
         
         if held
           puts
@@ -225,7 +240,7 @@ module Benchmark
           if hold? && @held_results && @held_results.key?(item.label)
            result = @held_results[item.label]
             create_report(item.label, result['measured_us'], result['iter'],
-              result['avg_ips'], result['sd_ips'], result['cycles'])
+                          create_stats(result['samples']), result['cycles'])
             next
           end
           
@@ -259,16 +274,13 @@ module Benchmark
 
           final_time = before
 
-          measured_us = measurements_us.inject(0) { |a,i| a + i }
+          measured_us = measurements_us.inject(:+)
 
-          all_ips = measurements_us.map { |time_us|
+          samples = measurements_us.map { |time_us|
             iterations_per_sec cycles, time_us
           }
 
-          avg_ips = Timing.mean(all_ips)
-          sd_ips =  Timing.stddev(all_ips, avg_ips).round
-
-          rep = create_report(item.label, measured_us, iter, avg_ips, sd_ips, cycles)
+          rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)
 
           if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
             rep.show_total_time!
@@ -284,8 +296,7 @@ module Benchmark
                 :item => item.label,
                 :measured_us => measured_us,
                 :iter => iter,
-                :avg_ips => avg_ips,
-                :sd_ips => sd_ips,
+                :samples => samples,
                 :cycles => cycles
               })
               f.write "\n"
@@ -302,6 +313,17 @@ module Benchmark
         false
       end
 
+      def create_stats(samples)
+        case @stats
+          when :sd
+            Stats::SD.new(samples)
+          when :bootstrap
+            Stats::Bootstrap.new(samples, @confidence)
+          else
+            raise "unknown stats #{@stats}"
+        end
+      end
+
       # Run comparison of entries in +@full_report+.
       def run_comparison
         @full_report.run_comparison if compare?
@@ -316,12 +338,11 @@ module Benchmark
       # @param label [String] Report item label.
       # @param measured_us [Integer] Measured time in microsecond.
       # @param iter [Integer] Iterations.
-      # @param avg_ips [Float] Average iterations per second.
-      # @param sd_ips [Float] Standard deviation iterations per second.
+      # @param samples [Array<Float>] Sampled iterations per second.
       # @param cycles [Integer] Number of Cycles.
       # @return [Report::Entry] Entry with data.
-      def create_report(label, measured_us, iter, avg_ips, sd_ips, cycles)
-        @full_report.add_entry label, measured_us, iter, avg_ips, sd_ips, cycles
+      def create_report(label, measured_us, iter, samples, cycles)
+        @full_report.add_entry label, measured_us, iter, samples, cycles
       end
     end
   end

data/lib/benchmark/ips/job/stdout_report.rb

@@ -27,6 +27,12 @@ module Benchmark
 
         def add_report(item, caller)
           $stdout.puts " #{item.body}"
+          @last_item = item
+        end
+
+        def footer
+          footer = @last_item.stats.footer
+          $stdout.puts footer.rjust(40) if footer
         end
 
         private

data/lib/benchmark/ips/report.rb

@@ -13,15 +13,13 @@ module Benchmark
         # @param [#to_s] label Label of entry.
         # @param [Integer] us Measured time in microsecond.
         # @param [Integer] iters Iterations.
-        # @param [Float] ips Iterations per second.
-        # @param [Float] ips_sd Standard deviation of iterations per second.
+        # @param [Object] stats Statistics.
         # @param [Integer] cycles Number of Cycles.
-        def initialize(label, us, iters, ips, ips_sd, cycles)
+        def initialize(label, us, iters, stats, cycles)
           @label = label
           @microseconds = us
           @iterations = iters
-          @ips = ips
-          @ips_sd = ips_sd
+          @stats = stats
           @measurement_cycle = cycles
           @show_total_time = false
         end
@@ -38,13 +36,9 @@ module Benchmark
         # @return [Integer] number of iterations.
         attr_reader :iterations
 
-        # Iterations per second.
-        # @return [Float] number of iterations per second.
-        attr_reader :ips
-
-        # Standard deviation of iteration per second.
-        # @return [Float] standard deviation of iteration per second.
-        attr_reader :ips_sd
+        # Statistical summary of samples.
+        # @return [Object] statisical summary.
+        attr_reader :stats
 
         # Number of Cycles.
         # @return [Integer] number of cycles.
@@ -65,8 +59,8 @@ module Benchmark
 
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
-        def stddev_percentage
-          100.0 * (@ips_sd.to_f / @ips.to_f)
+        def error_percentage
+          100.0 * (@stats.error.to_f / @stats.central_tendency)
         end
 
         alias_method :runtime, :seconds
@@ -78,7 +72,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(ips), stddev_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -87,7 +81,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
@@ -131,12 +125,11 @@ module Benchmark
       # @param label [String] Entry label.
       # @param microseconds [Integer] Measured time in microsecond.
       # @param iters [Integer] Iterations.
-      # @param ips [Float] Average Iterations per second.
-      # @param ips_sd [Float] Standard deviation of iterations per second.
+      # @param stats [Object] Statistical results.
       # @param measurement_cycle [Integer] Number of cycles.
       # @return [Report::Entry] Last added entry.
-      def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
-        entry = Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
+      def add_entry label, microseconds, iters, stats, measurement_cycle
+        entry = Entry.new(label, microseconds, iters, stats, measurement_cycle)
         @entries.delete_if { |e| e.label == label }
         @entries << entry
         entry
@@ -155,8 +148,10 @@ module Benchmark
         @data ||= @entries.collect do |entry|
           {
             :name => entry.label,
-            :ips =>  entry.ips,
-            :stddev => entry.ips_sd,
+            :central_tendency =>  entry.stats.central_tendency,
+            :ips =>  entry.stats.central_tendency, # for backwards compatibility
+            :error => entry.stats.error,
+            :stddev => entry.stats.error, # for backwards compatibility
             :microseconds => entry.microseconds,
             :iterations => entry.iterations,
             :cycles => entry.measurement_cycle,

data/lib/benchmark/timing.rb

@@ -8,7 +8,7 @@ module Benchmark
     # @param [Array] samples Samples to calculate mean.
     # @return [Float] Mean of given samples.
     def self.mean(samples)
-      sum = samples.inject(0) { |acc, i| acc + i }
+      sum = samples.inject(:+)
       sum / samples.size
     end
 
@@ -31,20 +31,6 @@ module Benchmark
       Math.sqrt variance(samples, m)
     end
 
-    # Resample mean of given samples.
-    # @param [Integer] resample_times Resample times, defaults to 100.
-    # @return [Array] Resampled samples.
-    def self.resample_mean(samples, resample_times=100)
-      resamples = []
-
-      resample_times.times do
-        resample = samples.map { samples[rand(samples.size)] }
-        resamples << Timing.mean(resample)
-      end
-
-      resamples
-    end
-
     # Recycle used objects by starting Garbage Collector.
     def self.clean_env
       # rbx
@@ -66,7 +52,7 @@ module Benchmark
 
       # Add one second to the time represenetation
       def self.add_second(t, s)
-        return t + (s * MICROSECONDS_PER_SECOND)
+        t + (s * MICROSECONDS_PER_SECOND)
       end
 
       # Return the number of microseconds between the 2 moments
@@ -81,7 +67,7 @@ module Benchmark
 
       # Add one second to the time represenetation
       def self.add_second(t, s)
-        return t + s
+        t + s
       end
 
       # Return the number of microseconds between the 2 moments

data/test/test_benchmark_ips.rb

@@ -55,11 +55,11 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep1.label
     assert_equal 4, rep1.iterations
-    assert_in_delta 4.0, rep1.ips, 0.2
+    assert_in_delta 4.0, rep1.stats.central_tendency, 0.2
 
     assert_equal "sleep 0.05", rep2.label
     assert_in_delta 20.0, rep2.iterations.to_f, 1.0
-    assert_in_delta 20.0, rep2.ips, 2.0
+    assert_in_delta 20.0, rep2.stats.central_tendency, 2.0
   end
 
   def test_ips_alternate_config
@@ -73,7 +73,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4, rep.iterations
-    assert_in_delta 4.0, rep.ips, 0.4
+    assert_in_delta 4.0, rep.stats.central_tendency, 0.4
   end
 
   def test_ips_old_config
@@ -85,7 +85,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4, rep.iterations
-    assert_in_delta 4.0, rep.ips, 0.2
+    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
   end
 
   def test_ips_config_suite
@@ -112,7 +112,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4*5, rep.iterations
-    assert_in_delta 4.0, rep.ips, 0.2
+    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
   end
 
   def test_ips_report_using_symbol
@@ -124,7 +124,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal :sleep_a_quarter_second, rep.label
     assert_equal 4*5, rep.iterations
-    assert_in_delta 4.0, rep.ips, 0.2
+    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
   end
 
   def test_ips_default_data

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.7.0
 platform: ruby
 authors:
 - Evan Phoenix
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-13 00:00:00.000000000 Z
+date: 2016-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.8'
+        version: '5.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.8'
+        version: '5.9'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement