benchmark-ips 2.7.1 → 2.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: dbf6e5f13ed635a69ab099631340497b0305df4e
-  data.tar.gz: 010de92f654e0dc25d1770b9ce83944540d71920
+SHA256:
+  metadata.gz: 574d0cb84279e02316df55c8d96f396b2a10301b15639ede54198eb278fd6ba2
+  data.tar.gz: d4ddabd9c093fd2c9559870818543d105c6fb2bbf3d38ad397b3bdf0df691397
 SHA512:
-  metadata.gz: 97c18c2d2bf77dd41320fdd5336b2aa562fc0db2b46f8a38866d087491c3d3a52b0e732b2169eaf330bb30e14822f01ff5d4f2f670d99a05624e30721c32bf53
-  data.tar.gz: 3cc170d620547e78039a1e007a2fe17aaa6dd0ac6c06d571b0e26aaab0933db69cde62cc63042d3fb6e2c0d2fb54135ea5f5b6c519923a0787284583f7564b4f
+  metadata.gz: 701868e1d0e33101daaf7e5005245b327dbf3b3d02fb19a8e89eb39563540b5ca5b1ff91c4e3cd4bd9dd54f97537ca48e668608742ee0569a13336818ddfef23
+  data.tar.gz: a032682bd26691ad5d4a837ba956b9acf50e45c69036e07bdda347791f78bf388cb3339873fd6d5ca9cbf6edeec5090c15ae6fa56a712f6d0f0340729ce8e792

data/History.txt

@@ -1,3 +1,41 @@
+=== 2.8.4 / 2020-12-03
+
+* Bug fix
+  * Fixed hold! when results file does not exist.
+
+=== 2.8.3 / 2020-08-28
+
+* Bug fix 
+  * Fixed inaccuracy caused by integer overflows.
+
+=== 2.8.2 / 2020-05-04
+
+* Bug fix 
+  * Fixed problems with Manifest.txt.
+  * Empty interim results files are ignored.
+
+=== 2.8.0 / 2020-05-01
+
+* Feature
+  * Allow running with empty ips block.
+  * Added save! method for saving interim results.
+  * Run more than just 1 cycle during warmup to reduce overhead.
+  * Optimized Job::Entry hot-path for fairer results on JRuby/TruffleRuby.
+
+* Bug fix
+  * Removed the warmup section if set to 0.
+  * Added some RDoc docs.
+  * Added some examples in examples/
+
+=== 2.7.2 / 2016-08-18
+
+* 1 bug fix:
+  * Restore old accessors. Fixes #76
+
+=== 2.7.1 / 2016-08-08
+
+Add missing files
+
 === 2.7.0 / 2016-08-05
 
 * 1 minor features:

data/Manifest.txt

@@ -1,5 +1,4 @@
 .autotest
-Gemfile.lock
 History.txt
 Manifest.txt
 README.md
@@ -13,5 +12,6 @@ lib/benchmark/ips/report.rb
 lib/benchmark/ips/share.rb
 lib/benchmark/ips/stats/bootstrap.rb
 lib/benchmark/ips/stats/sd.rb
+lib/benchmark/ips/stats/stats_metric.rb
 lib/benchmark/timing.rb
 test/test_benchmark_ips.rb

data/README.md

@@ -1,13 +1,14 @@
+# benchmark-ips
+
+* rdoc :: http://rubydoc.info/gems/benchmark-ips
+* home :: https://github.com/evanphx/benchmark-ips
+
 [![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
 [![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
 [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
 
-# benchmark-ips
-
 * https://github.com/evanphx/benchmark-ips
 
-* [documentation](http://rubydoc.info/gems/benchmark-ips)
-
 ## DESCRIPTION:
 
 An iterations per second enhancement to Benchmark.
@@ -155,6 +156,11 @@ This will run only one benchmarks each time you run the command, storing
 results in the specified file. The file is deleted when all results have been
 gathered and the report is shown.
 
+Alternatively, if you prefer a different approach, the `save!` command is
+available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
+the `examples/` directory.
+
+
 ### Multiple iterations
 
 In some cases you may want to run multiple iterations of the warmup and

data/lib/benchmark/compare.rb

@@ -40,18 +40,14 @@ module Benchmark
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
+      $stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
-        
+
         $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
-        
-        best_low = best.stats.central_tendency - best.stats.error
-        report_high = report.stats.central_tendency + report.stats.error
-        overlaps = report_high > best_low 
-        
-        if overlaps
+
+        if report.stats.overlaps?(best.stats)
           $stdout.print "same-ish: difference falls within error"
         else
           slowdown, error = report.stats.slowdown(best.stats)
@@ -61,7 +57,7 @@ module Benchmark
           end
           $stdout.print " slower"
         end
-        
+
         $stdout.puts
       end
 

data/lib/benchmark/ips.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/stats_metric'
 require 'benchmark/ips/stats/sd'
 require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
@@ -15,10 +16,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.7.1"
+    VERSION = "2.8.4"
 
     # CODENAME of current version.
-    CODENAME = "Cultivating Confidence"
+    CODENAME = "Tardy Turtle"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -54,10 +55,17 @@ module Benchmark
 
       yield job
 
-      job.load_held_results if job.hold? && job.held_results?
+      job.load_held_results
 
       job.run
 
+      if job.run_single? && job.all_results_have_been_run?
+        job.clear_held_results
+      else
+        job.save_held_results
+        puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
+      end
+
       $stdout.sync = sync
       job.run_comparison
       job.generate_json
@@ -102,4 +110,68 @@ module Benchmark
   end
 
   extend Benchmark::IPS # make ips available as module-level method
+
+  ##
+  # :singleton-method: ips
+  #
+  #     require 'benchmark/ips'
+  #
+  #     Benchmark.ips do |x|
+  #       # Configure the number of seconds used during
+  #       # the warmup phase (default 2) and calculation phase (default 5)
+  #       x.config(:time => 5, :warmup => 2)
+  #
+  #       # These parameters can also be configured this way
+  #       x.time = 5
+  #       x.warmup = 2
+  #
+  #       # Typical mode, runs the block as many times as it can
+  #       x.report("addition") { 1 + 2 }
+  #
+  #       # To reduce overhead, the number of iterations is passed in
+  #       # and the block must run the code the specific number of times.
+  #       # Used for when the workload is very small and any overhead
+  #       # introduces incorrectable errors.
+  #       x.report("addition2") do |times|
+  #         i = 0
+  #         while i < times
+  #           1 + 2
+  #           i += 1
+  #         end
+  #       end
+  #
+  #       # To reduce overhead even more, grafts the code given into
+  #       # the loop that performs the iterations internally to reduce
+  #       # overhead. Typically not needed, use the |times| form instead.
+  #       x.report("addition3", "1 + 2")
+  #
+  #       # Really long labels should be formatted correctly
+  #       x.report("addition-test-long-label") { 1 + 2 }
+  #
+  #       # Compare the iterations per second of the various reports!
+  #       x.compare!
+  #     end
+  #
+  # This will generate the following report:
+  #
+  #     Calculating -------------------------------------
+  #                 addition    71.254k i/100ms
+  #                addition2    68.658k i/100ms
+  #                addition3    83.079k i/100ms
+  #     addition-test-long-label
+  #                             70.129k i/100ms
+  #     -------------------------------------------------
+  #                 addition     4.955M (± 8.7%) i/s -     24.155M
+  #                addition2    24.011M (± 9.5%) i/s -    114.246M
+  #                addition3    23.958M (±10.1%) i/s -    115.064M
+  #     addition-test-long-label
+  #                              5.014M (± 9.1%) i/s -     24.545M
+  #
+  #     Comparison:
+  #                addition2: 24011974.8 i/s
+  #                addition3: 23958619.8 i/s - 1.00x slower
+  #     addition-test-long-label:  5014756.0 i/s - 4.79x slower
+  #                 addition:  4955278.9 i/s - 4.85x slower
+  #
+  # See also Benchmark::IPS
 end

data/lib/benchmark/ips/job.rb

@@ -9,6 +9,7 @@ module Benchmark
       # The percentage of the expected runtime to allow
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
+      POW_2_30 = 1 << 30
 
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
@@ -58,11 +59,12 @@ module Benchmark
         @stdout = opts[:quiet] ? nil : StdoutReport.new
         @list = []
         @compare = false
+        @run_single = false
         @json_path = false
         @held_path = nil
         @held_results = nil
 
-        @timing = {}
+        @timing = Hash.new 1 # default to 1 in case warmup isn't run
         @full_report = Report.new
 
         # Default warmup and calculation time in seconds.
@@ -94,7 +96,7 @@ module Benchmark
         @compare
       end
 
-      # Set @compare to true.
+      # Run comparison utility.
       def compare!
         @compare = true
       end
@@ -105,9 +107,27 @@ module Benchmark
         !!@held_path
       end
 
-      # Set @hold to true.
+      # Hold after each iteration.
+      # @param held_path [String] File name to store hold file.
       def hold!(held_path)
         @held_path = held_path
+        @run_single = true
+      end
+
+      # Save interim results. Similar to hold, but all reports are run
+      # The report label must change for each invocation.
+      # One way to achieve this is to include the version in the label.
+      # @param held_path [String] File name to store hold file.
+      def save!(held_path)
+        @held_path = held_path
+        @run_single = false
+      end
+
+      # Return true if items are to be run one at a time.
+      # For the traditional hold, this is true
+      # @return [Boolean] Run just a single item?
+      def run_single?
+        @run_single
       end
 
       # Return true if job needs to generate json.
@@ -116,7 +136,7 @@ module Benchmark
         !!@json_path
       end
 
-      # Set @json_path to given path, defaults to "data.json".
+      # Generate json to given path, defaults to "data.json".
       def json!(path="data.json")
         @json_path = path
       end
@@ -166,84 +186,112 @@ module Benchmark
       def iterations_per_sec cycles, time_us
         MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
       end
-      
-      def held_results?
-        File.exist?(@held_path)
-      end
-      
+
       def load_held_results
+        return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
+        require "json"
+        @held_results = {}
+        JSON.load(IO.read(@held_path)).each do |result|
+          @held_results[result['item']] = result
+          create_report(result['item'], result['measured_us'], result['iter'],
+                        create_stats(result['samples']), result['cycles'])
+        end
+      end
+
+      def save_held_results
+        return unless @held_path
         require "json"
-        @held_results = Hash[File.open(@held_path).map { |line|
-          result = JSON.parse(line)
-          [result['item'], result]
-        }]
+        data = full_report.entries.map { |e|
+          {
+            'item' => e.label,
+            'measured_us' => e.microseconds,
+            'iter' => e.iterations,
+            'samples' => e.samples,
+            'cycles' => e.measurement_cycle
+          }
+        }
+        IO.write(@held_path, JSON.generate(data) << "\n")
+      end
+
+      def all_results_have_been_run?
+        @full_report.entries.size == @list.size
       end
-      
+
+      def clear_held_results
+        File.delete @held_path if File.exist?(@held_path)
+      end
+
       def run
-        @stdout.start_warming if @stdout
-        @iterations.times do
-          run_warmup
+        if @warmup && @warmup != 0 then
+          @stdout.start_warming if @stdout
+          @iterations.times do
+            run_warmup
+          end
         end
-        
+
         @stdout.start_running if @stdout
-        
-        held = nil
-        
+
         @iterations.times do |n|
-          held = run_benchmark
+          run_benchmark
         end
 
         @stdout.footer if @stdout
-        
-        if held
-          puts
-          puts 'Pausing here -- run Ruby again to measure the next benchmark...'
-        end
       end
 
       # Run warmup.
       def run_warmup
         @list.each do |item|
-          next if hold? && @held_results && @held_results.key?(item.label)
-          
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
           @suite.warming item.label, @warmup if @suite
           @stdout.warming item.label, @warmup if @stdout
 
           Timing.clean_env
 
+          # Run for up to half of the configured warmup time with an increasing
+          # number of cycles to reduce overhead and improve accuracy.
+          # This also avoids running with a constant number of cycles, which a
+          # JIT might speculate on and then have to recompile in #run_benchmark.
           before = Timing.now
-          target = Timing.add_second before, @warmup
+          target = Timing.add_second before, @warmup / 2.0
 
-          warmup_iter = 0
-
-          while Timing.now < target
-            item.call_times(1)
-            warmup_iter += 1
+          cycles = 1
+          warmup_iter = 1
+          warmup_time_us = 0.0
+          while Timing.now + warmup_time_us * 2 < target
+            t0 = Timing.now
+            item.call_times cycles
+            t1 = Timing.now
+            warmup_iter = cycles
+            warmup_time_us = Timing.time_us(t0, t1)
+
+            # If the number of cycles would go outside the 32-bit signed integers range
+            # then exit the loop to avoid overflows and start the 100ms warmup runs
+            break if cycles >= POW_2_30
+            cycles *= 2
           end
 
-          after = Timing.now
+          cycles = cycles_per_100ms warmup_time_us, warmup_iter
+          @timing[item] = cycles
 
-          warmup_time_us = Timing.time_us(before, after)
-
-          @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
+          # Run for the remaining of warmup in a similar way as #run_benchmark.
+          target = Timing.add_second before, @warmup
+          while Timing.now + MICROSECONDS_PER_100MS < target
+            item.call_times cycles
+          end
 
           @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
           @suite.warmup_stats warmup_time_us, @timing[item] if @suite
-          
-          break if hold?
+
+          break if run_single?
         end
       end
 
       # Run calculation.
       def run_benchmark
         @list.each do |item|
-          if hold? && @held_results && @held_results.key?(item.label)
-           result = @held_results[item.label]
-            create_report(item.label, result['measured_us'], result['iter'],
-                          create_stats(result['samples']), result['cycles'])
-            next
-          end
-          
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
           @suite.running item.label, @time if @suite
           @stdout.running item.label, @time if @stdout
 
@@ -257,7 +305,7 @@ module Benchmark
           cycles = @timing[item]
 
           target = Timing.add_second Timing.now, @time
-          
+
           while (before = Timing.now) < target
             item.call_times cycles
             after = Timing.now
@@ -288,29 +336,9 @@ module Benchmark
 
           @stdout.add_report rep, caller(1).first if @stdout
           @suite.add_report rep, caller(1).first if @suite
-          
-          if hold? && item != @list.last
-            File.open @held_path, "a" do |f|
-              require "json"
-              f.write JSON.generate({
-                :item => item.label,
-                :measured_us => measured_us,
-                :iter => iter,
-                :samples => samples,
-                :cycles => cycles
-              })
-              f.write "\n"
-            end
-            
-            return true
-          end
-        end
-        
-        if hold? && @full_report.entries.size == @list.size
-          File.delete @held_path if File.exist?(@held_path)
+
+          break if run_single?
         end
-        
-        false
       end
 
       def create_stats(samples)

data/lib/benchmark/ips/job/entry.rb

@@ -11,10 +11,12 @@ module Benchmark
         def initialize(label, action)
           @label = label
 
+          # We define #call_times on the singleton class of each Entry instance.
+          # That way, there is no polymorphism for `@action.call` inside #call_times.
+
           if action.kind_of? String
-            compile action
+            compile_string action
             @action = self
-            @as_action = true
           else
             unless action.respond_to? :call
               raise ArgumentError, "invalid action, must respond to #call"
@@ -23,12 +25,10 @@ module Benchmark
             @action = action
 
             if action.respond_to? :arity and action.arity > 0
-              @call_loop = true
+              compile_block_with_manual_loop
             else
-              @call_loop = false
+              compile_block
             end
-
-            @as_action = false
           end
         end
 
@@ -40,25 +40,43 @@ module Benchmark
         # @return [String, Proc] Code to be called, could be String / Proc.
         attr_reader :action
 
-        # Call action by given times, return if +@call_loop+ is present.
+        # Call action by given times.
         # @param times [Integer] Times to call +@action+.
         # @return [Integer] Number of times the +@action+ has been called.
         def call_times(times)
-          return @action.call(times) if @call_loop
+          raise '#call_times should be redefined per Benchmark::IPS::Job::Entry instance'
+        end
 
-          act = @action
+        def compile_block
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              act = @action
 
-          i = 0
-          while i < times
-            act.call
-            i += 1
-          end
+              i = 0
+              while i < times
+                act.call
+                i += 1
+              end
+            end
+          CODE
+          m.class_eval code
+        end
+
+        def compile_block_with_manual_loop
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              @action.call(times)
+            end
+          CODE
+          m.class_eval code
         end
 
         # Compile code into +call_times+ method.
         # @param str [String] Code to be compiled.
         # @return [Symbol] :call_times.
-        def compile(str)
+        def compile_string(str)
           m = (class << self; self; end)
           code = <<-CODE
             def call_times(__total);

data/lib/benchmark/ips/job/stdout_report.rb

@@ -2,10 +2,14 @@ module Benchmark
   module IPS
     class Job
       class StdoutReport
+        def initialize
+          @last_item = nil
+        end
+
         def start_warming
           $stdout.puts "Warming up --------------------------------------"
         end
-        
+
         def start_running
           $stdout.puts "Calculating -------------------------------------"
         end
@@ -31,6 +35,7 @@ module Benchmark
         end
 
         def footer
+          return unless @last_item
           footer = @last_item.stats.footer
           $stdout.puts footer.rjust(40) if footer
         end

data/lib/benchmark/ips/report.rb

@@ -40,6 +40,22 @@ module Benchmark
         # @return [Object] statisical summary.
         attr_reader :stats
 
+        # LEGACY: Iterations per second.
+        # @return [Float] number of iterations per second.
+        def ips
+          @stats.central_tendency
+        end
+
+        # LEGACY: Standard deviation of iteration per second.
+        # @return [Float] standard deviation of iteration per second.
+        def ips_sd
+          @stats.error
+        end
+
+        def samples
+          @stats.samples
+        end
+
         # Number of Cycles.
         # @return [Integer] number of cycles.
         attr_reader :measurement_cycle
@@ -60,7 +76,7 @@ module Benchmark
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
         def error_percentage
-          100.0 * (@stats.error.to_f / @stats.central_tendency)
+          @stats.error_percentage
         end
 
         alias_method :runtime, :seconds
@@ -72,7 +88,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -81,7 +97,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])

data/lib/benchmark/ips/stats/bootstrap.rb

@@ -3,27 +3,30 @@ module Benchmark
     module Stats
 
       class Bootstrap
-
-        attr_reader :data
+        include StatsMetric
+        attr_reader :data, :error, :samples
 
         def initialize(samples, confidence)
           dependencies
           @iterations = 10_000
           @confidence = (confidence / 100.0).to_s
+          @samples = samples
           @data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
           interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
           @median = interval.median
           @error = interval.error
         end
 
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @median
         end
 
-        def error
-          @error
-        end
-
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
           low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
           error = Timing.mean([slowdown - low, high - slowdown])

data/lib/benchmark/ips/stats/sd.rb

@@ -1,33 +1,41 @@
 module Benchmark
   module IPS
     module Stats
-      
+
       class SD
-        
+        include StatsMetric
+        attr_reader :error, :samples
+
         def initialize(samples)
+          @samples = samples
           @mean = Timing.mean(samples)
           @error = Timing.stddev(samples, @mean).round
         end
-        
+
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @mean
         end
-        
-        def error
-          @error
-        end
 
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
-          slowdown = baseline.central_tendency.to_f / central_tendency
-          [slowdown, nil]
+          if baseline.central_tendency > central_tendency
+            [baseline.central_tendency.to_f / central_tendency, 0]
+          else
+            [central_tendency.to_f / baseline.central_tendency, 0]
+          end
         end
 
         def footer
           nil
         end
-        
+
       end
-    
+
     end
   end
 end

data/lib/benchmark/ips/stats/stats_metric.rb

@@ -0,0 +1,21 @@
+module Benchmark
+  module IPS
+    module Stats
+      module StatsMetric
+        # Return entry's standard deviation of iteration per second in percentage.
+        # @return [Float] +@ips_sd+ in percentage.
+        def error_percentage
+          100.0 * (error.to_f / central_tendency)
+        end
+
+        def overlaps?(baseline)
+          baseline_low = baseline.central_tendency - baseline.error
+          baseline_high = baseline.central_tendency + baseline.error
+          my_high = central_tendency + error
+          my_low  = central_tendency - error
+          my_high > baseline_low && my_low < baseline_high
+        end
+      end
+    end
+  end
+end

data/test/test_benchmark_ips.rb

@@ -1,6 +1,7 @@
 require "minitest/autorun"
 require "benchmark/ips"
 require "stringio"
+require "tmpdir"
 
 class TestBenchmarkIPS < Minitest::Test
   def setup
@@ -20,6 +21,19 @@ class TestBenchmarkIPS < Minitest::Test
     assert $stdout.string.size > 0
   end
 
+  def test_warmup0
+    $stdout = @old_stdout
+
+    out, err = capture_io do
+      Benchmark.ips(:time => 1, :warmup => 0, :quiet => false) do |x|
+        x.report("sleep 0.25") { sleep(0.25) }
+      end
+    end
+
+    refute_match(/Warming up -+/, out)
+    assert_empty err
+  end
+
   def test_output
     Benchmark.ips(1) do |x|
       x.report("operation") { 100 * 100 }
@@ -55,11 +69,11 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep1.label
     assert_equal 4, rep1.iterations
-    assert_in_delta 4.0, rep1.stats.central_tendency, 0.2
+    assert_in_delta 4.0, rep1.ips, 0.2
 
     assert_equal "sleep 0.05", rep2.label
     assert_in_delta 20.0, rep2.iterations.to_f, 1.0
-    assert_in_delta 20.0, rep2.stats.central_tendency, 2.0
+    assert_in_delta 20.0, rep2.ips, 2.0
   end
 
   def test_ips_alternate_config
@@ -73,7 +87,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4, rep.iterations
-    assert_in_delta 4.0, rep.stats.central_tendency, 0.4
+    assert_in_delta 4.0, rep.ips, 0.4
   end
 
   def test_ips_old_config
@@ -85,7 +99,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4, rep.iterations
-    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
+    assert_in_delta 4.0, rep.ips, 0.2
   end
 
   def test_ips_config_suite
@@ -112,7 +126,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal "sleep 0.25", rep.label
     assert_equal 4*5, rep.iterations
-    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
+    assert_in_delta 4.0, rep.ips, 0.2
   end
 
   def test_ips_report_using_symbol
@@ -124,7 +138,7 @@ class TestBenchmarkIPS < Minitest::Test
 
     assert_equal :sleep_a_quarter_second, rep.label
     assert_equal 4*5, rep.iterations
-    assert_in_delta 4.0, rep.stats.central_tendency, 0.2
+    assert_in_delta 4.0, rep.ips, 0.2
   end
 
   def test_ips_default_data
@@ -140,6 +154,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert all_data[0][:stddev]
   end
 
+  def test_ips_empty
+    report = Benchmark.ips do |_x|
+
+    end
+
+    all_data = report.data
+
+    assert all_data
+    assert_equal [], all_data
+  end
+
   def test_json_output
     json_file = Tempfile.new("data.json")
 
@@ -158,4 +183,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert data[0]["ips"]
     assert data[0]["stddev"]
   end
+
+  def test_hold!
+    temp_file_name = Dir::Tmpname.create(["benchmark-ips", ".tmp"]) { }
+
+    Benchmark.ips(:time => 0.001, :warmup => 0.001) do |x|
+      x.report("operation") { 100 * 100 }
+      x.report("operation2") { 100 * 100 }
+      x.hold! temp_file_name
+    end
+
+    assert File.exist?(temp_file_name)
+    File.unlink(temp_file_name)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.7.1
+  version: 2.8.4
 platform: ruby
 authors:
 - Evan Phoenix
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2020-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -16,42 +16,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '5.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '5.14'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.22'
 description: An iterations per second enhancement to Benchmark.
 email:
 - evan@phx.io
@@ -63,7 +69,6 @@ extra_rdoc_files:
 - README.md
 files:
 - ".autotest"
-- Gemfile.lock
 - History.txt
 - Manifest.txt
 - README.md
@@ -77,13 +82,15 @@ files:
 - lib/benchmark/ips/share.rb
 - lib/benchmark/ips/stats/bootstrap.rb
 - lib/benchmark/ips/stats/sd.rb
+- lib/benchmark/ips/stats/stats_metric.rb
 - lib/benchmark/timing.rb
 - test/test_benchmark_ips.rb
 homepage: https://github.com/evanphx/benchmark-ips
 licenses:
 - MIT
-metadata: {}
-post_install_message: 
+metadata:
+  homepage_uri: https://github.com/evanphx/benchmark-ips
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -100,9 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: An iterations per second enhancement to Benchmark.
 test_files: []

data/Gemfile.lock

@@ -1,18 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    hoe (3.15.0)
-      rake (>= 0.8, < 12.0)
-    minitest (5.8.4)
-    rake (10.5.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  hoe (~> 3.14)
-  minitest
-  rake (~> 10.5)
-
-BUNDLED WITH
-   1.11.2