benchmark-ips 2.7.2 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: feceba6a7a761d8eecc66d7a4e66e3458d0cd5fa
-  data.tar.gz: 5c905cacd3d45fa76abf30ac2695745f1c3a23ac
+SHA256:
+  metadata.gz: 5bec7b23527f8a5f331788f9de05073eede35e26bbc5a6b81c05a609d964c0ff
+  data.tar.gz: aa8ba81457df018d94262d0eb722870d34406204d02a61b0ad1aa398fa2903ff
 SHA512:
-  metadata.gz: 5d5bb801ae291bab7c026f0495d460259aa5fdde347ae776630f0c8f1fe24f994da1222005c87d1cd1f4ac130edf8438c8e92a5e46d94b7a6a47022ca2fb2275
-  data.tar.gz: 13c64a8eb5e8aac398b3b6f258a323907e643cd9c45e1bfd6438b72fedfe55c798d6ee32489a480135ce92d919c453907bee9f468c977a9903cd06cfefc40211
+  metadata.gz: 0b8c313bd722fd52229c280d4d566fb9e246b8ae4253a0f508a5b9ca7fffd35cd780cc6c9aa96894733ee87d0df844feae530c10dbf38989cb07367094ce19f3
+  data.tar.gz: 037a07ffd3f129a06a68b5c53617379a9c7e3397fc9b02a961c7e7bb9b566f3ad0b423a43b3b5ed3e40f9edfac4f6eed810bc231c8e6d1398afaa65730d62946

data/History.txt

@@ -1,3 +1,38 @@
+=== 2.9.0 / 2021-05-21
+
+* Features
+  * Suite can now be set via an accessor
+  * Default SHARE_URL is now `ips.fastruby.io`, operated by Ombu Labs.
+
+=== 2.8.4 / 2020-12-03
+
+* Bug fix
+  * Fixed hold! when results file does not exist.
+
+=== 2.8.3 / 2020-08-28
+
+* Bug fix
+  * Fixed inaccuracy caused by integer overflows.
+
+=== 2.8.2 / 2020-05-04
+
+* Bug fix
+  * Fixed problems with Manifest.txt.
+  * Empty interim results files are ignored.
+
+=== 2.8.0 / 2020-05-01
+
+* Feature
+  * Allow running with empty ips block.
+  * Added save! method for saving interim results.
+  * Run more than just 1 cycle during warmup to reduce overhead.
+  * Optimized Job::Entry hot-path for fairer results on JRuby/TruffleRuby.
+
+* Bug fix
+  * Removed the warmup section if set to 0.
+  * Added some RDoc docs.
+  * Added some examples in examples/
+
 === 2.7.2 / 2016-08-18
 
 * 1 bug fix:

data/Manifest.txt

@@ -1,5 +1,4 @@
 .autotest
-Gemfile.lock
 History.txt
 Manifest.txt
 README.md
@@ -13,5 +12,6 @@ lib/benchmark/ips/report.rb
 lib/benchmark/ips/share.rb
 lib/benchmark/ips/stats/bootstrap.rb
 lib/benchmark/ips/stats/sd.rb
+lib/benchmark/ips/stats/stats_metric.rb
 lib/benchmark/timing.rb
 test/test_benchmark_ips.rb

data/README.md

@@ -1,13 +1,14 @@
+# benchmark-ips
+
+* rdoc :: http://rubydoc.info/gems/benchmark-ips
+* home :: https://github.com/evanphx/benchmark-ips
+
 [![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
 [![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
 [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
 
-# benchmark-ips
-
 * https://github.com/evanphx/benchmark-ips
 
-* [documentation](http://rubydoc.info/gems/benchmark-ips)
-
 ## DESCRIPTION:
 
 An iterations per second enhancement to Benchmark.
@@ -155,6 +156,11 @@ This will run only one benchmarks each time you run the command, storing
 results in the specified file. The file is deleted when all results have been
 gathered and the report is shown.
 
+Alternatively, if you prefer a different approach, the `save!` command is
+available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
+the `examples/` directory.
+
+
 ### Multiple iterations
 
 In some cases you may want to run multiple iterations of the warmup and
@@ -180,11 +186,15 @@ end
 
 ### Online sharing
 
-If you want to share quickly your benchmark result with others. Run you benchmark
-with `SHARE=1` argument. I.e.: `SHARE=1 ruby my_benchmark.rb`.
-Result will be sent to [benchmark.fyi](https://benchmark.fyi/) and benchmark-ips
+If you want to quickly share your benchmark result with others, run you benchmark
+with `SHARE=1` argument. For example: `SHARE=1 ruby my_benchmark.rb`.
+
+Result will be sent to [benchmark.fyi](https://ips.fastruby.io/) and benchmark-ips
 will display the link to share the benchmark's result.
 
+If you want to run your own instance of [benchmark.fyi](https://github.com/evanphx/benchmark.fyi)
+and share it to that instance, you can do this: `SHARE_URL=https://ips.example.com ruby my_benchmark.rb`
+
 ### Advanced Statistics
 
 By default, the margin of error shown is plus-minus one standard deviation. If
@@ -221,7 +231,7 @@ Benchmark.ips do |x|
 
   x.stats = :bootstrap
   x.confidence = 95
-  
+
   # confidence is 95% by default, so it can be omitted
 
 end

data/lib/benchmark/compare.rb

@@ -40,18 +40,14 @@ module Benchmark
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
+      $stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
-        
+
         $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
-        
-        best_low = best.stats.central_tendency - best.stats.error
-        report_high = report.stats.central_tendency + report.stats.error
-        overlaps = report_high > best_low 
-        
-        if overlaps
+
+        if report.stats.overlaps?(best.stats)
           $stdout.print "same-ish: difference falls within error"
         else
           slowdown, error = report.stats.slowdown(best.stats)
@@ -61,7 +57,7 @@ module Benchmark
           end
           $stdout.print " slower"
         end
-        
+
         $stdout.puts
       end
 

data/lib/benchmark/ips.rb

@@ -1,11 +1,14 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/stats_metric'
 require 'benchmark/ips/stats/sd'
 require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
+require 'benchmark/ips/noop_suite'
 require 'benchmark/ips/job/entry'
 require 'benchmark/ips/job/stdout_report'
+require 'benchmark/ips/job/noop_report'
 require 'benchmark/ips/job'
 
 # Performance benchmarking library
@@ -15,10 +18,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.7.2"
+    VERSION = "2.9.0"
 
     # CODENAME of current version.
-    CODENAME = "Cultivating Confidence"
+    CODENAME = "Sleepy Sasquatch"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -32,32 +35,30 @@ module Benchmark
         time, warmup, quiet = args
       end
 
-      suite = nil
-
       sync, $stdout.sync = $stdout.sync, true
 
-      if defined? Benchmark::Suite and Suite.current
-        suite = Benchmark::Suite.current
-      end
-
-      quiet ||= (suite && suite.quiet?)
-
-      job = Job.new({:suite => suite,
-                     :quiet => quiet
-      })
+      job = Job.new
 
       job_opts = {}
       job_opts[:time] = time unless time.nil?
       job_opts[:warmup] = warmup unless warmup.nil?
+      job_opts[:quiet] = quiet unless quiet.nil?
 
       job.config job_opts
 
       yield job
 
-      job.load_held_results if job.hold? && job.held_results?
+      job.load_held_results
 
       job.run
 
+      if job.run_single? && job.all_results_have_been_run?
+        job.clear_held_results
+      else
+        job.save_held_results
+        puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
+      end
+
       $stdout.sync = sync
       job.run_comparison
       job.generate_json
@@ -102,4 +103,68 @@ module Benchmark
   end
 
   extend Benchmark::IPS # make ips available as module-level method
+
+  ##
+  # :singleton-method: ips
+  #
+  #     require 'benchmark/ips'
+  #
+  #     Benchmark.ips do |x|
+  #       # Configure the number of seconds used during
+  #       # the warmup phase (default 2) and calculation phase (default 5)
+  #       x.config(:time => 5, :warmup => 2)
+  #
+  #       # These parameters can also be configured this way
+  #       x.time = 5
+  #       x.warmup = 2
+  #
+  #       # Typical mode, runs the block as many times as it can
+  #       x.report("addition") { 1 + 2 }
+  #
+  #       # To reduce overhead, the number of iterations is passed in
+  #       # and the block must run the code the specific number of times.
+  #       # Used for when the workload is very small and any overhead
+  #       # introduces incorrectable errors.
+  #       x.report("addition2") do |times|
+  #         i = 0
+  #         while i < times
+  #           1 + 2
+  #           i += 1
+  #         end
+  #       end
+  #
+  #       # To reduce overhead even more, grafts the code given into
+  #       # the loop that performs the iterations internally to reduce
+  #       # overhead. Typically not needed, use the |times| form instead.
+  #       x.report("addition3", "1 + 2")
+  #
+  #       # Really long labels should be formatted correctly
+  #       x.report("addition-test-long-label") { 1 + 2 }
+  #
+  #       # Compare the iterations per second of the various reports!
+  #       x.compare!
+  #     end
+  #
+  # This will generate the following report:
+  #
+  #     Calculating -------------------------------------
+  #                 addition    71.254k i/100ms
+  #                addition2    68.658k i/100ms
+  #                addition3    83.079k i/100ms
+  #     addition-test-long-label
+  #                             70.129k i/100ms
+  #     -------------------------------------------------
+  #                 addition     4.955M (± 8.7%) i/s -     24.155M
+  #                addition2    24.011M (± 9.5%) i/s -    114.246M
+  #                addition3    23.958M (±10.1%) i/s -    115.064M
+  #     addition-test-long-label
+  #                              5.014M (± 9.1%) i/s -     24.545M
+  #
+  #     Comparison:
+  #                addition2: 24011974.8 i/s
+  #                addition3: 23958619.8 i/s - 1.00x slower
+  #     addition-test-long-label:  5014756.0 i/s - 4.79x slower
+  #                 addition:  4955278.9 i/s - 4.85x slower
+  #
+  # See also Benchmark::IPS
 end

data/lib/benchmark/ips/job.rb

@@ -9,6 +9,7 @@ module Benchmark
       # The percentage of the expected runtime to allow
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
+      POW_2_30 = 1 << 30
 
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
@@ -50,19 +51,24 @@ module Benchmark
       # @return [Integer]
       attr_accessor :confidence
 
+      # Silence output
+      # @return [Boolean]
+      attr_reader :quiet
+
+      # Suite
+      # @return [Benchmark::IPS::NoopSuite]
+      attr_reader :suite
+
       # Instantiate the Benchmark::IPS::Job.
-      # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
-      # @option opts [Boolean] (false) :quiet Suppress the printing of information.
       def initialize opts={}
-        @suite = opts[:suite] || nil
-        @stdout = opts[:quiet] ? nil : StdoutReport.new
         @list = []
-        @compare = false
+        @run_single = false
         @json_path = false
+        @compare = false
         @held_path = nil
         @held_results = nil
 
-        @timing = {}
+        @timing = Hash.new 1 # default to 1 in case warmup isn't run
         @full_report = Report.new
 
         # Default warmup and calculation time in seconds.
@@ -86,6 +92,20 @@ module Benchmark
         @iterations = opts[:iterations] if opts[:iterations]
         @stats = opts[:stats] if opts[:stats]
         @confidence = opts[:confidence] if opts[:confidence]
+        self.quiet = opts[:quiet]
+        self.suite = opts[:suite]
+      end
+
+      def quiet=(val)
+        @stdout = reporter(quiet: val)
+      end
+
+      def suite=(suite)
+        @suite = suite || Benchmark::IPS::NoopSuite.new
+      end
+
+      def reporter(quiet:)
+        quiet ? NoopReport.new : StdoutReport.new
       end
 
       # Return true if job needs to be compared.
@@ -94,7 +114,7 @@ module Benchmark
         @compare
       end
 
-      # Set @compare to true.
+      # Run comparison utility.
       def compare!
         @compare = true
       end
@@ -105,9 +125,27 @@ module Benchmark
         !!@held_path
       end
 
-      # Set @hold to true.
+      # Hold after each iteration.
+      # @param held_path [String] File name to store hold file.
       def hold!(held_path)
         @held_path = held_path
+        @run_single = true
+      end
+
+      # Save interim results. Similar to hold, but all reports are run
+      # The report label must change for each invocation.
+      # One way to achieve this is to include the version in the label.
+      # @param held_path [String] File name to store hold file.
+      def save!(held_path)
+        @held_path = held_path
+        @run_single = false
+      end
+
+      # Return true if items are to be run one at a time.
+      # For the traditional hold, this is true
+      # @return [Boolean] Run just a single item?
+      def run_single?
+        @run_single
       end
 
       # Return true if job needs to generate json.
@@ -116,7 +154,7 @@ module Benchmark
         !!@json_path
       end
 
-      # Set @json_path to given path, defaults to "data.json".
+      # Generate json to given path, defaults to "data.json".
       def json!(path="data.json")
         @json_path = path
       end
@@ -166,86 +204,114 @@ module Benchmark
       def iterations_per_sec cycles, time_us
         MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
       end
-      
-      def held_results?
-        File.exist?(@held_path)
-      end
-      
+
       def load_held_results
+        return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
         require "json"
-        @held_results = Hash[File.open(@held_path).map { |line|
-          result = JSON.parse(line)
-          [result['item'], result]
-        }]
+        @held_results = {}
+        JSON.load(IO.read(@held_path)).each do |result|
+          @held_results[result['item']] = result
+          create_report(result['item'], result['measured_us'], result['iter'],
+                        create_stats(result['samples']), result['cycles'])
+        end
+      end
+
+      def save_held_results
+        return unless @held_path
+        require "json"
+        data = full_report.entries.map { |e|
+          {
+            'item' => e.label,
+            'measured_us' => e.microseconds,
+            'iter' => e.iterations,
+            'samples' => e.samples,
+            'cycles' => e.measurement_cycle
+          }
+        }
+        IO.write(@held_path, JSON.generate(data) << "\n")
       end
-      
+
+      def all_results_have_been_run?
+        @full_report.entries.size == @list.size
+      end
+
+      def clear_held_results
+        File.delete @held_path if File.exist?(@held_path)
+      end
+
       def run
-        @stdout.start_warming if @stdout
-        @iterations.times do
-          run_warmup
+        if @warmup && @warmup != 0 then
+          @stdout.start_warming
+          @iterations.times do
+            run_warmup
+          end
         end
-        
-        @stdout.start_running if @stdout
-        
-        held = nil
-        
+
+        @stdout.start_running
+
         @iterations.times do |n|
-          held = run_benchmark
+          run_benchmark
         end
 
-        @stdout.footer if @stdout
-        
-        if held
-          puts
-          puts 'Pausing here -- run Ruby again to measure the next benchmark...'
-        end
+        @stdout.footer
       end
 
       # Run warmup.
       def run_warmup
         @list.each do |item|
-          next if hold? && @held_results && @held_results.key?(item.label)
-          
-          @suite.warming item.label, @warmup if @suite
-          @stdout.warming item.label, @warmup if @stdout
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
+          @suite.warming item.label, @warmup
+          @stdout.warming item.label, @warmup
 
           Timing.clean_env
 
+          # Run for up to half of the configured warmup time with an increasing
+          # number of cycles to reduce overhead and improve accuracy.
+          # This also avoids running with a constant number of cycles, which a
+          # JIT might speculate on and then have to recompile in #run_benchmark.
           before = Timing.now
-          target = Timing.add_second before, @warmup
+          target = Timing.add_second before, @warmup / 2.0
 
-          warmup_iter = 0
-
-          while Timing.now < target
-            item.call_times(1)
-            warmup_iter += 1
+          cycles = 1
+          warmup_iter = 1
+          warmup_time_us = 0.0
+          while Timing.now + warmup_time_us * 2 < target
+            t0 = Timing.now
+            item.call_times cycles
+            t1 = Timing.now
+            warmup_iter = cycles
+            warmup_time_us = Timing.time_us(t0, t1)
+
+            # If the number of cycles would go outside the 32-bit signed integers range
+            # then exit the loop to avoid overflows and start the 100ms warmup runs
+            break if cycles >= POW_2_30
+            cycles *= 2
           end
 
-          after = Timing.now
+          cycles = cycles_per_100ms warmup_time_us, warmup_iter
+          @timing[item] = cycles
 
-          warmup_time_us = Timing.time_us(before, after)
+          # Run for the remaining of warmup in a similar way as #run_benchmark.
+          target = Timing.add_second before, @warmup
+          while Timing.now + MICROSECONDS_PER_100MS < target
+            item.call_times cycles
+          end
 
-          @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
+          @stdout.warmup_stats warmup_time_us, @timing[item]
+          @suite.warmup_stats warmup_time_us, @timing[item]
 
-          @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
-          @suite.warmup_stats warmup_time_us, @timing[item] if @suite
-          
-          break if hold?
+          break if run_single?
         end
       end
 
       # Run calculation.
       def run_benchmark
         @list.each do |item|
-          if hold? && @held_results && @held_results.key?(item.label)
-           result = @held_results[item.label]
-            create_report(item.label, result['measured_us'], result['iter'],
-                          create_stats(result['samples']), result['cycles'])
-            next
-          end
-          
-          @suite.running item.label, @time if @suite
-          @stdout.running item.label, @time if @stdout
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
+          @suite.running item.label, @time
+          @stdout.running item.label, @time
 
           Timing.clean_env
 
@@ -257,7 +323,7 @@ module Benchmark
           cycles = @timing[item]
 
           target = Timing.add_second Timing.now, @time
-          
+
           while (before = Timing.now) < target
             item.call_times cycles
             after = Timing.now
@@ -286,31 +352,11 @@ module Benchmark
             rep.show_total_time!
           end
 
-          @stdout.add_report rep, caller(1).first if @stdout
-          @suite.add_report rep, caller(1).first if @suite
-          
-          if hold? && item != @list.last
-            File.open @held_path, "a" do |f|
-              require "json"
-              f.write JSON.generate({
-                :item => item.label,
-                :measured_us => measured_us,
-                :iter => iter,
-                :samples => samples,
-                :cycles => cycles
-              })
-              f.write "\n"
-            end
-            
-            return true
-          end
-        end
-        
-        if hold? && @full_report.entries.size == @list.size
-          File.delete @held_path if File.exist?(@held_path)
+          @stdout.add_report rep, caller(1).first
+          @suite.add_report rep, caller(1).first
+
+          break if run_single?
         end
-        
-        false
       end
 
       def create_stats(samples)

data/lib/benchmark/ips/job/entry.rb

@@ -11,10 +11,12 @@ module Benchmark
         def initialize(label, action)
           @label = label
 
+          # We define #call_times on the singleton class of each Entry instance.
+          # That way, there is no polymorphism for `@action.call` inside #call_times.
+
           if action.kind_of? String
-            compile action
+            compile_string action
             @action = self
-            @as_action = true
           else
             unless action.respond_to? :call
               raise ArgumentError, "invalid action, must respond to #call"
@@ -23,12 +25,10 @@ module Benchmark
             @action = action
 
             if action.respond_to? :arity and action.arity > 0
-              @call_loop = true
+              compile_block_with_manual_loop
             else
-              @call_loop = false
+              compile_block
             end
-
-            @as_action = false
           end
         end
 
@@ -40,25 +40,43 @@ module Benchmark
         # @return [String, Proc] Code to be called, could be String / Proc.
         attr_reader :action
 
-        # Call action by given times, return if +@call_loop+ is present.
+        # Call action by given times.
         # @param times [Integer] Times to call +@action+.
         # @return [Integer] Number of times the +@action+ has been called.
         def call_times(times)
-          return @action.call(times) if @call_loop
+          raise '#call_times should be redefined per Benchmark::IPS::Job::Entry instance'
+        end
 
-          act = @action
+        def compile_block
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              act = @action
 
-          i = 0
-          while i < times
-            act.call
-            i += 1
-          end
+              i = 0
+              while i < times
+                act.call
+                i += 1
+              end
+            end
+          CODE
+          m.class_eval code
+        end
+
+        def compile_block_with_manual_loop
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              @action.call(times)
+            end
+          CODE
+          m.class_eval code
         end
 
         # Compile code into +call_times+ method.
         # @param str [String] Code to be compiled.
         # @return [Symbol] :call_times.
-        def compile(str)
+        def compile_string(str)
           m = (class << self; self; end)
           code = <<-CODE
             def call_times(__total);

data/lib/benchmark/ips/job/stdout_report.rb

@@ -2,10 +2,14 @@ module Benchmark
   module IPS
     class Job
       class StdoutReport
+        def initialize
+          @last_item = nil
+        end
+
         def start_warming
           $stdout.puts "Warming up --------------------------------------"
         end
-        
+
         def start_running
           $stdout.puts "Calculating -------------------------------------"
         end
@@ -31,6 +35,7 @@ module Benchmark
         end
 
         def footer
+          return unless @last_item
           footer = @last_item.stats.footer
           $stdout.puts footer.rjust(40) if footer
         end

data/lib/benchmark/ips/report.rb

@@ -52,6 +52,10 @@ module Benchmark
           @stats.error
         end
 
+        def samples
+          @stats.samples
+        end
+
         # Number of Cycles.
         # @return [Integer] number of cycles.
         attr_reader :measurement_cycle
@@ -72,7 +76,7 @@ module Benchmark
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
         def error_percentage
-          100.0 * (@stats.error.to_f / @stats.central_tendency)
+          @stats.error_percentage
         end
 
         alias_method :runtime, :seconds
@@ -84,7 +88,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -93,7 +97,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])

data/lib/benchmark/ips/share.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'net/https'
 require 'json'
@@ -5,7 +7,7 @@ require 'json'
 module Benchmark
   module IPS
     class Share
-      DEFAULT_URL = "https://benchmark.fyi"
+      DEFAULT_URL = "https://ips.fastruby.io"
       def initialize(report, job)
         @report = report
         @job = job

data/lib/benchmark/ips/stats/bootstrap.rb

@@ -3,27 +3,30 @@ module Benchmark
     module Stats
 
       class Bootstrap
-
-        attr_reader :data
+        include StatsMetric
+        attr_reader :data, :error, :samples
 
         def initialize(samples, confidence)
           dependencies
           @iterations = 10_000
           @confidence = (confidence / 100.0).to_s
+          @samples = samples
           @data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
           interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
           @median = interval.median
           @error = interval.error
         end
 
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @median
         end
 
-        def error
-          @error
-        end
-
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
           low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
           error = Timing.mean([slowdown - low, high - slowdown])

data/lib/benchmark/ips/stats/sd.rb

@@ -1,33 +1,41 @@
 module Benchmark
   module IPS
     module Stats
-      
+
       class SD
-        
+        include StatsMetric
+        attr_reader :error, :samples
+
         def initialize(samples)
+          @samples = samples
           @mean = Timing.mean(samples)
           @error = Timing.stddev(samples, @mean).round
         end
-        
+
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @mean
         end
-        
-        def error
-          @error
-        end
 
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
-          slowdown = baseline.central_tendency.to_f / central_tendency
-          [slowdown, nil]
+          if baseline.central_tendency > central_tendency
+            [baseline.central_tendency.to_f / central_tendency, 0]
+          else
+            [central_tendency.to_f / baseline.central_tendency, 0]
+          end
         end
 
         def footer
           nil
         end
-        
+
       end
-    
+
     end
   end
 end

data/lib/benchmark/ips/stats/stats_metric.rb

@@ -0,0 +1,21 @@
+module Benchmark
+  module IPS
+    module Stats
+      module StatsMetric
+        # Return entry's standard deviation of iteration per second in percentage.
+        # @return [Float] +@ips_sd+ in percentage.
+        def error_percentage
+          100.0 * (error.to_f / central_tendency)
+        end
+
+        def overlaps?(baseline)
+          baseline_low = baseline.central_tendency - baseline.error
+          baseline_high = baseline.central_tendency + baseline.error
+          my_high = central_tendency + error
+          my_low  = central_tendency - error
+          my_high > baseline_low && my_low < baseline_high
+        end
+      end
+    end
+  end
+end

data/test/test_benchmark_ips.rb

@@ -1,6 +1,7 @@
 require "minitest/autorun"
 require "benchmark/ips"
 require "stringio"
+require "tmpdir"
 
 class TestBenchmarkIPS < Minitest::Test
   def setup
@@ -20,6 +21,19 @@ class TestBenchmarkIPS < Minitest::Test
     assert $stdout.string.size > 0
   end
 
+  def test_warmup0
+    $stdout = @old_stdout
+
+    out, err = capture_io do
+      Benchmark.ips(:time => 1, :warmup => 0, :quiet => false) do |x|
+        x.report("sleep 0.25") { sleep(0.25) }
+      end
+    end
+
+    refute_match(/Warming up -+/, out)
+    assert_empty err
+  end
+
   def test_output
     Benchmark.ips(1) do |x|
       x.report("operation") { 100 * 100 }
@@ -40,6 +54,13 @@ class TestBenchmarkIPS < Minitest::Test
     end
 
     assert $stdout.string.size.zero?
+
+    Benchmark.ips do |x|
+      x.quiet = true
+      x.report("operation") { 100 * 100 }
+    end
+
+    assert $stdout.string.size.zero?
   end
 
   def test_ips
@@ -103,6 +124,21 @@ class TestBenchmarkIPS < Minitest::Test
     assert_equal [:warming, :warmup_stats, :running, :add_report], suite.calls
   end
 
+  def test_ips_config_suite_by_accsr
+    suite = Struct.new(:calls) do
+      def method_missing(method, *args)
+        calls << method
+      end
+    end.new([])
+
+    Benchmark.ips(0.1, 0.1) do |x|
+      x.suite = suite
+      x.report("job") {}
+    end
+
+    assert_equal [:warming, :warmup_stats, :running, :add_report], suite.calls
+  end
+
   def test_ips_defaults
     report = Benchmark.ips do |x|
       x.report("sleep 0.25") { sleep(0.25) }
@@ -140,6 +176,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert all_data[0][:stddev]
   end
 
+  def test_ips_empty
+    report = Benchmark.ips do |_x|
+
+    end
+
+    all_data = report.data
+
+    assert all_data
+    assert_equal [], all_data
+  end
+
   def test_json_output
     json_file = Tempfile.new("data.json")
 
@@ -158,4 +205,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert data[0]["ips"]
     assert data[0]["stddev"]
   end
+
+  def test_hold!
+    temp_file_name = Dir::Tmpname.create(["benchmark-ips", ".tmp"]) { }
+
+    Benchmark.ips(:time => 0.001, :warmup => 0.001) do |x|
+      x.report("operation") { 100 * 100 }
+      x.report("operation2") { 100 * 100 }
+      x.hold! temp_file_name
+    end
+
+    assert File.exist?(temp_file_name)
+    File.unlink(temp_file_name)
+  end
 end

metadata

@@ -1,57 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.9.0
 platform: ruby
 authors:
 - Evan Phoenix
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2021-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
-  type: :development
+        version: '5.14'
+  name: minitest
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '5.14'
 - !ruby/object:Gem::Dependency
-  name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
-  type: :development
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  name: rdoc
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
-  name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
-  type: :development
+        version: '3.22'
+  name: hoe
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.22'
 description: An iterations per second enhancement to Benchmark.
 email:
 - evan@phx.io
@@ -63,7 +69,6 @@ extra_rdoc_files:
 - README.md
 files:
 - ".autotest"
-- Gemfile.lock
 - History.txt
 - Manifest.txt
 - README.md
@@ -77,13 +82,15 @@ files:
 - lib/benchmark/ips/share.rb
 - lib/benchmark/ips/stats/bootstrap.rb
 - lib/benchmark/ips/stats/sd.rb
+- lib/benchmark/ips/stats/stats_metric.rb
 - lib/benchmark/timing.rb
 - test/test_benchmark_ips.rb
 homepage: https://github.com/evanphx/benchmark-ips
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/evanphx/benchmark-ips
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -100,9 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: An iterations per second enhancement to Benchmark.
 test_files: []

data/Gemfile.lock

@@ -1,18 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    hoe (3.15.0)
-      rake (>= 0.8, < 12.0)
-    minitest (5.8.4)
-    rake (10.5.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  hoe (~> 3.14)
-  minitest
-  rake (~> 10.5)
-
-BUNDLED WITH
-   1.11.2