benchmark-ips 2.7.2 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: feceba6a7a761d8eecc66d7a4e66e3458d0cd5fa
-  data.tar.gz: 5c905cacd3d45fa76abf30ac2695745f1c3a23ac
+SHA256:
+  metadata.gz: 54899cb6088746f3b585d3e471795fa63c5fa06b9a452e7923d5211ea096eff1
+  data.tar.gz: 21f3a97a2971d77dbf33fa05a497f7c61499497da66b8aff26987c64d6e2de3c
 SHA512:
-  metadata.gz: 5d5bb801ae291bab7c026f0495d460259aa5fdde347ae776630f0c8f1fe24f994da1222005c87d1cd1f4ac130edf8438c8e92a5e46d94b7a6a47022ca2fb2275
-  data.tar.gz: 13c64a8eb5e8aac398b3b6f258a323907e643cd9c45e1bfd6438b72fedfe55c798d6ee32489a480135ce92d919c453907bee9f468c977a9903cd06cfefc40211
+  metadata.gz: 3b7dd9271181dc58b32864d0d60ecf58c8daf92dc024ab764bc07289a39650db30354d72fc8f586207c6f5ccfaccf72c41eb9ac75d4e7be7af7a18c60f6e7860
+  data.tar.gz: 11731f19a4fbc1abcc6e31ad6c5d4ea752f26d3a5ce9573b4e07011b568c455abfbd5a53b5e41408d7e931a5327d1c6c0a2200c25d10897b9d4111d45326c7d5

data/Manifest.txt

@@ -1,5 +1,4 @@
 .autotest
-Gemfile.lock
 History.txt
 Manifest.txt
 README.md

data/README.md

@@ -1,13 +1,14 @@
+# benchmark-ips
+
+* rdoc :: http://rubydoc.info/gems/benchmark-ips
+* home :: https://github.com/evanphx/benchmark-ips
+
 [![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
 [![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
 [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
 
-# benchmark-ips
-
 * https://github.com/evanphx/benchmark-ips
 
-* [documentation](http://rubydoc.info/gems/benchmark-ips)
-
 ## DESCRIPTION:
 
 An iterations per second enhancement to Benchmark.
@@ -155,6 +156,11 @@ This will run only one benchmarks each time you run the command, storing
 results in the specified file. The file is deleted when all results have been
 gathered and the report is shown.
 
+Alternatively, if you prefer a different approach, the `save!` command is
+available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
+the `examples/` directory.
+
+
 ### Multiple iterations
 
 In some cases you may want to run multiple iterations of the warmup and

data/lib/benchmark/compare.rb

@@ -40,18 +40,14 @@ module Benchmark
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
+      $stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
-        
+
         $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
-        
-        best_low = best.stats.central_tendency - best.stats.error
-        report_high = report.stats.central_tendency + report.stats.error
-        overlaps = report_high > best_low 
-        
-        if overlaps
+
+        if report.stats.overlaps?(best.stats)
           $stdout.print "same-ish: difference falls within error"
         else
           slowdown, error = report.stats.slowdown(best.stats)
@@ -61,7 +57,7 @@ module Benchmark
           end
           $stdout.print " slower"
         end
-        
+
         $stdout.puts
       end
 

data/lib/benchmark/ips.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/stats_metric'
 require 'benchmark/ips/stats/sd'
 require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
@@ -15,10 +16,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.7.2"
+    VERSION = "2.8.0"
 
     # CODENAME of current version.
-    CODENAME = "Cultivating Confidence"
+    CODENAME = "Tardy Turtle"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -54,10 +55,17 @@ module Benchmark
 
       yield job
 
-      job.load_held_results if job.hold? && job.held_results?
+      job.load_held_results
 
       job.run
 
+      if job.run_single? && job.all_results_have_been_run?
+        job.clear_held_results
+      else
+        job.save_held_results
+        puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
+      end
+
       $stdout.sync = sync
       job.run_comparison
       job.generate_json
@@ -101,5 +109,68 @@ module Benchmark
     end
   end
 
+  ##
+  # :singleton-method: ips
+  #
+  #     require 'benchmark/ips'
+  #
+  #     Benchmark.ips do |x|
+  #       # Configure the number of seconds used during
+  #       # the warmup phase (default 2) and calculation phase (default 5)
+  #       x.config(:time => 5, :warmup => 2)
+  #
+  #       # These parameters can also be configured this way
+  #       x.time = 5
+  #       x.warmup = 2
+  #
+  #       # Typical mode, runs the block as many times as it can
+  #       x.report("addition") { 1 + 2 }
+  #
+  #       # To reduce overhead, the number of iterations is passed in
+  #       # and the block must run the code the specific number of times.
+  #       # Used for when the workload is very small and any overhead
+  #       # introduces incorrectable errors.
+  #       x.report("addition2") do |times|
+  #         i = 0
+  #         while i < times
+  #           1 + 2
+  #           i += 1
+  #         end
+  #       end
+  #
+  #       # To reduce overhead even more, grafts the code given into
+  #       # the loop that performs the iterations internally to reduce
+  #       # overhead. Typically not needed, use the |times| form instead.
+  #       x.report("addition3", "1 + 2")
+  #
+  #       # Really long labels should be formatted correctly
+  #       x.report("addition-test-long-label") { 1 + 2 }
+  #
+  #       # Compare the iterations per second of the various reports!
+  #       x.compare!
+  #     end
+  #
+  # This will generate the following report:
+  #
+  #     Calculating -------------------------------------
+  #                 addition    71.254k i/100ms
+  #                addition2    68.658k i/100ms
+  #                addition3    83.079k i/100ms
+  #     addition-test-long-label
+  #                             70.129k i/100ms
+  #     -------------------------------------------------
+  #                 addition     4.955M (± 8.7%) i/s -     24.155M
+  #                addition2    24.011M (± 9.5%) i/s -    114.246M
+  #                addition3    23.958M (±10.1%) i/s -    115.064M
+  #     addition-test-long-label
+  #                              5.014M (± 9.1%) i/s -     24.545M
+  #
+  #     Comparison:
+  #                addition2: 24011974.8 i/s
+  #                addition3: 23958619.8 i/s - 1.00x slower
+  #     addition-test-long-label:  5014756.0 i/s - 4.79x slower
+  #                 addition:  4955278.9 i/s - 4.85x slower
+  #
+  # See also Benchmark::IPS
   extend Benchmark::IPS # make ips available as module-level method
 end

data/lib/benchmark/ips/job.rb

@@ -58,11 +58,12 @@ module Benchmark
         @stdout = opts[:quiet] ? nil : StdoutReport.new
         @list = []
         @compare = false
+        @run_single = false
         @json_path = false
         @held_path = nil
         @held_results = nil
 
-        @timing = {}
+        @timing = Hash.new 1 # default to 1 in case warmup isn't run
         @full_report = Report.new
 
         # Default warmup and calculation time in seconds.
@@ -94,7 +95,7 @@ module Benchmark
         @compare
       end
 
-      # Set @compare to true.
+      # Run comparison utility.
       def compare!
         @compare = true
       end
@@ -105,9 +106,27 @@ module Benchmark
         !!@held_path
       end
 
-      # Set @hold to true.
+      # Hold after each iteration.
+      # @param held_path [String] File name to store hold file.
       def hold!(held_path)
         @held_path = held_path
+        @run_single = true
+      end
+
+      # Save interim results. Similar to hold, but all reports are run
+      # The report label must change for each invocation.
+      # One way to achieve this is to include the version in the label.
+      # @param held_path [String] File name to store hold file.
+      def save!(held_path)
+        @held_path = held_path
+        @run_single = false
+      end
+
+      # Return true if items are to be run one at a time.
+      # For the traditional hold, this is true
+      # @return [Boolean] Run just a single item?
+      def run_single?
+        @run_single
       end
 
       # Return true if job needs to generate json.
@@ -116,7 +135,7 @@ module Benchmark
         !!@json_path
       end
 
-      # Set @json_path to given path, defaults to "data.json".
+      # Generate json to given path, defaults to "data.json".
       def json!(path="data.json")
         @json_path = path
       end
@@ -166,84 +185,108 @@ module Benchmark
       def iterations_per_sec cycles, time_us
         MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
       end
-      
-      def held_results?
-        File.exist?(@held_path)
-      end
-      
+
       def load_held_results
+        return unless @held_path && File.exist?(@held_path)
+        require "json"
+        @held_results = {}
+        JSON.load(IO.read(@held_path)).each do |result|
+          @held_results[result['item']] = result
+          create_report(result['item'], result['measured_us'], result['iter'],
+                        create_stats(result['samples']), result['cycles'])
+        end
+      end
+
+      def save_held_results
+        return unless @held_path
         require "json"
-        @held_results = Hash[File.open(@held_path).map { |line|
-          result = JSON.parse(line)
-          [result['item'], result]
-        }]
+        data = full_report.entries.map { |e|
+          {
+            'item' => e.label,
+            'measured_us' => e.microseconds,
+            'iter' => e.iterations,
+            'samples' => e.samples,
+            'cycles' => e.measurement_cycle
+          }
+        }
+        IO.write(@held_path, JSON.generate(data) << "\n")
+      end
+
+      def all_results_have_been_run?
+        @full_report.entries.size == @list.size
       end
-      
+
+      def clear_held_results
+        File.delete @held_path if File.exist?(@held_path)
+      end
+
       def run
-        @stdout.start_warming if @stdout
-        @iterations.times do
-          run_warmup
+        if @warmup && @warmup != 0 then
+          @stdout.start_warming if @stdout
+          @iterations.times do
+            run_warmup
+          end
         end
-        
+
         @stdout.start_running if @stdout
-        
-        held = nil
-        
+
         @iterations.times do |n|
-          held = run_benchmark
+          run_benchmark
         end
 
         @stdout.footer if @stdout
-        
-        if held
-          puts
-          puts 'Pausing here -- run Ruby again to measure the next benchmark...'
-        end
       end
 
       # Run warmup.
       def run_warmup
         @list.each do |item|
-          next if hold? && @held_results && @held_results.key?(item.label)
-          
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
           @suite.warming item.label, @warmup if @suite
           @stdout.warming item.label, @warmup if @stdout
 
           Timing.clean_env
 
+          # Run for up to half of the configured warmup time with an increasing
+          # number of cycles to reduce overhead and improve accuracy.
+          # This also avoids running with a constant number of cycles, which a
+          # JIT might speculate on and then have to recompile in #run_benchmark.
           before = Timing.now
-          target = Timing.add_second before, @warmup
+          target = Timing.add_second before, @warmup / 2.0
 
-          warmup_iter = 0
-
-          while Timing.now < target
-            item.call_times(1)
-            warmup_iter += 1
+          cycles = 1
+          warmup_iter = 1
+          warmup_time_us = 0.0
+          while Timing.now + warmup_time_us * 2 < target
+            t0 = Timing.now
+            item.call_times cycles
+            t1 = Timing.now
+            warmup_iter = cycles
+            warmup_time_us = Timing.time_us(t0, t1)
+            cycles *= 2
           end
 
-          after = Timing.now
+          cycles = cycles_per_100ms warmup_time_us, warmup_iter
+          @timing[item] = cycles
 
-          warmup_time_us = Timing.time_us(before, after)
-
-          @timing[item] = cycles_per_100ms warmup_time_us, warmup_iter
+          # Run for the remaining of warmup in a similar way as #run_benchmark.
+          target = Timing.add_second before, @warmup
+          while Timing.now + MICROSECONDS_PER_100MS < target
+            item.call_times cycles
+          end
 
           @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
           @suite.warmup_stats warmup_time_us, @timing[item] if @suite
-          
-          break if hold?
+
+          break if run_single?
         end
       end
 
       # Run calculation.
       def run_benchmark
         @list.each do |item|
-          if hold? && @held_results && @held_results.key?(item.label)
-           result = @held_results[item.label]
-            create_report(item.label, result['measured_us'], result['iter'],
-                          create_stats(result['samples']), result['cycles'])
-            next
-          end
-          
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
           @suite.running item.label, @time if @suite
           @stdout.running item.label, @time if @stdout
 
@@ -257,7 +300,7 @@ module Benchmark
           cycles = @timing[item]
 
           target = Timing.add_second Timing.now, @time
-          
+
           while (before = Timing.now) < target
             item.call_times cycles
             after = Timing.now
@@ -288,29 +331,9 @@ module Benchmark
 
           @stdout.add_report rep, caller(1).first if @stdout
           @suite.add_report rep, caller(1).first if @suite
-          
-          if hold? && item != @list.last
-            File.open @held_path, "a" do |f|
-              require "json"
-              f.write JSON.generate({
-                :item => item.label,
-                :measured_us => measured_us,
-                :iter => iter,
-                :samples => samples,
-                :cycles => cycles
-              })
-              f.write "\n"
-            end
-            
-            return true
-          end
-        end
-        
-        if hold? && @full_report.entries.size == @list.size
-          File.delete @held_path if File.exist?(@held_path)
+
+          break if run_single?
         end
-        
-        false
       end
 
       def create_stats(samples)

data/lib/benchmark/ips/job/stdout_report.rb

@@ -2,10 +2,14 @@ module Benchmark
   module IPS
     class Job
       class StdoutReport
+        def initialize
+          @last_item = nil
+        end
+
         def start_warming
           $stdout.puts "Warming up --------------------------------------"
         end
-        
+
         def start_running
           $stdout.puts "Calculating -------------------------------------"
         end
@@ -31,6 +35,7 @@ module Benchmark
         end
 
         def footer
+          return unless @last_item
           footer = @last_item.stats.footer
           $stdout.puts footer.rjust(40) if footer
         end

data/lib/benchmark/ips/report.rb

@@ -52,6 +52,10 @@ module Benchmark
           @stats.error
         end
 
+        def samples
+          @stats.samples
+        end
+
         # Number of Cycles.
         # @return [Integer] number of cycles.
         attr_reader :measurement_cycle
@@ -72,7 +76,7 @@ module Benchmark
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
         def error_percentage
-          100.0 * (@stats.error.to_f / @stats.central_tendency)
+          @stats.error_percentage
         end
 
         alias_method :runtime, :seconds
@@ -84,7 +88,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -93,7 +97,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])

data/lib/benchmark/ips/stats/bootstrap.rb

@@ -3,27 +3,30 @@ module Benchmark
     module Stats
 
       class Bootstrap
-
-        attr_reader :data
+        include StatsMetric
+        attr_reader :data, :error, :samples
 
         def initialize(samples, confidence)
           dependencies
           @iterations = 10_000
           @confidence = (confidence / 100.0).to_s
+          @samples = samples
           @data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
           interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
           @median = interval.median
           @error = interval.error
         end
 
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @median
         end
 
-        def error
-          @error
-        end
-
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
           low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
           error = Timing.mean([slowdown - low, high - slowdown])

data/lib/benchmark/ips/stats/sd.rb

@@ -1,33 +1,41 @@
 module Benchmark
   module IPS
     module Stats
-      
+
       class SD
-        
+        include StatsMetric
+        attr_reader :error, :samples
+
         def initialize(samples)
+          @samples = samples
           @mean = Timing.mean(samples)
           @error = Timing.stddev(samples, @mean).round
         end
-        
+
+        # Average stat value
+        # @return [Float] central_tendency
         def central_tendency
           @mean
         end
-        
-        def error
-          @error
-        end
 
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
         def slowdown(baseline)
-          slowdown = baseline.central_tendency.to_f / central_tendency
-          [slowdown, nil]
+          if baseline.central_tendency > central_tendency
+            [baseline.central_tendency.to_f / central_tendency, 0]
+          else
+            [central_tendency.to_f / baseline.central_tendency, 0]
+          end
         end
 
         def footer
           nil
         end
-        
+
       end
-    
+
     end
   end
 end

data/test/test_benchmark_ips.rb

@@ -20,6 +20,19 @@ class TestBenchmarkIPS < Minitest::Test
     assert $stdout.string.size > 0
   end
 
+  def test_warmup0
+    $stdout = @old_stdout
+
+    out, err = capture_io do
+      Benchmark.ips(:time => 1, :warmup => 0, :quiet => false) do |x|
+        x.report("sleep 0.25") { sleep(0.25) }
+      end
+    end
+
+    refute_match(/Warming up -+/, out)
+    assert_empty err
+  end
+
   def test_output
     Benchmark.ips(1) do |x|
       x.report("operation") { 100 * 100 }
@@ -140,6 +153,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert all_data[0][:stddev]
   end
 
+  def test_ips_empty
+    report = Benchmark.ips do |_x|
+
+    end
+
+    all_data = report.data
+
+    assert all_data
+    assert_equal [], all_data
+  end
+
   def test_json_output
     json_file = Tempfile.new("data.json")
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.7.2
+  version: 2.8.0
 platform: ruby
 authors:
 - Evan Phoenix
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2020-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -16,42 +16,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '5.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.9'
+        version: '5.14'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.15'
+        version: '3.22'
 description: An iterations per second enhancement to Benchmark.
 email:
 - evan@phx.io
@@ -63,7 +69,6 @@ extra_rdoc_files:
 - README.md
 files:
 - ".autotest"
-- Gemfile.lock
 - History.txt
 - Manifest.txt
 - README.md
@@ -82,7 +87,8 @@ files:
 homepage: https://github.com/evanphx/benchmark-ips
 licenses:
 - MIT
-metadata: {}
+metadata:
+  homepage_uri: https://github.com/evanphx/benchmark-ips
 post_install_message: 
 rdoc_options:
 - "--main"
@@ -100,8 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: An iterations per second enhancement to Benchmark.

data/Gemfile.lock

@@ -1,18 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    hoe (3.15.0)
-      rake (>= 0.8, < 12.0)
-    minitest (5.8.4)
-    rake (10.5.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  hoe (~> 3.14)
-  minitest
-  rake (~> 10.5)
-
-BUNDLED WITH
-   1.11.2