benchmark-ips 2.13.0 → 2.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc54b5cf0d24b23822486adf5759cceeefbef9a60c780454318472f4739050f6
-  data.tar.gz: ea2764b060fef8c931c4636eb63e84d48f98242f7335648fed8b82b2fdeb053d
+  metadata.gz: 941e6893612b6307694098b25ebe3cc688dac27020c98cf76d603ead0bd72299
+  data.tar.gz: 622715bcc49a8e11c1c6285b35745b9115417e1055b8dc394ca6f5ba07186b66
 SHA512:
-  metadata.gz: 72ed2d83e42b125ca812aa1e79113be4726f8bb2e0ca4cb784dddc1d70eddc16c088308e1214cba9f7ec83ad77d89d9ae544b7892b6abac8f3e0b88d1617e7a8
-  data.tar.gz: 94bda95d5db4a9032692e732a04eb3147ad3839b0e674f13d676b1fdf2130a8f0a269e6f5817791683965bf86662be10c7e5476e0539d71916b6b35562ac7806
+  metadata.gz: b0e5c2205809523369329ea675f095dc62cefa330e6adb6005b12a6ff4f4d5cfd38ffc16fe819761e4ed10a2c41b1100af8304c3a038383d2377700436873bbb
+  data.tar.gz: 318217090715cb2a541606ef8ab91087aec9238ffa0d6b29d5b2b1eeebc2e5d0a725856b59ded62081f1135f2a8bb916c25a31d604c0824accaf67e46ecfab8f

data/History.md

@@ -1,3 +1,14 @@
+### 2.14.0 / 2024-09-08
+
+* Feature
+  * Adds Benchmark::IPS.quick_compare.
+  * Adds absolute duration of each iteration to compare output.
+
+### 2.13.0 / 2023-12-12
+
+* Feature
+  * Prints ruby version at top of report.
+
 ### 2.12.0 / 2023-03-08
 
 * Feature
@@ -187,7 +198,7 @@ Add missing files
 * 1 minor fix:
   * Don't send label through printf so that % work directly
 
-* 1 documenation changes:
+* 1 documentation changes:
   * Use HEREDOC and wrap at 80 chars for example result description
 
 * 1 usage fix:

data/README.md

@@ -4,7 +4,7 @@
 * home :: https://github.com/evanphx/benchmark-ips
 
 [![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
-[![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
+[![CI](https://github.com/evanphx/benchmark-ips/actions/workflows/ci.yml/badge.svg)](https://github.com/evanphx/benchmark-ips/actions/workflows/ci.yml)
 [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
 
 * https://github.com/evanphx/benchmark-ips
@@ -27,11 +27,7 @@ require 'benchmark/ips'
 Benchmark.ips do |x|
   # Configure the number of seconds used during
   # the warmup phase (default 2) and calculation phase (default 5)
-  x.config(:time => 5, :warmup => 2)
-
-  # These parameters can also be configured this way
-  x.time = 5
-  x.warmup = 2
+  x.config(warmup: 2, time: 5)
 
   # Typical mode, runs the block as many times as it can
   x.report("addition") { 1 + 2 }
@@ -43,8 +39,9 @@ Benchmark.ips do |x|
   x.report("addition2") do |times|
     i = 0
     while i < times
-      1 + 2
       i += 1
+
+      1 + 2
     end
   end
 
@@ -64,24 +61,24 @@ end
 This will generate the following report:
 
 ```
-Calculating -------------------------------------
-            addition    71.254k i/100ms
-           addition2    68.658k i/100ms
-           addition3    83.079k i/100ms
+Warming up --------------------------------------
+            addition     3.572M i/100ms
+           addition2     3.672M i/100ms
+           addition3     3.677M i/100ms
 addition-test-long-label
-                        70.129k i/100ms
--------------------------------------------------
-            addition     4.955M (± 8.7%) i/s -     24.155M
-           addition2    24.011M (± 9.5%) i/s -    114.246M
-           addition3    23.958M (±10.1%) i/s -    115.064M
+                         3.511M i/100ms
+Calculating -------------------------------------
+            addition     36.209M (± 2.8%) i/s   (27.62 ns/i) -    182.253M in   5.037433s
+           addition2     36.552M (± 7.8%) i/s   (27.36 ns/i) -    183.541M in   5.069987s
+           addition3     36.639M (± 4.8%) i/s   (27.29 ns/i) -    182.994M in   5.009234s
 addition-test-long-label
-                         5.014M (± 9.1%) i/s -     24.545M
+                         36.164M (± 5.8%) i/s   (27.65 ns/i) -    181.312M in   5.038364s
 
 Comparison:
-           addition2: 24011974.8 i/s
-           addition3: 23958619.8 i/s - 1.00x slower
-addition-test-long-label:  5014756.0 i/s - 4.79x slower
-            addition:  4955278.9 i/s - 4.85x slower
+           addition2: 36558904.5 i/s
+           addition3: 36359284.0 i/s - same-ish: difference falls within error
+addition-test-long-label: 36135428.8 i/s - same-ish: difference falls within error
+            addition: 34666931.3 i/s - same-ish: difference falls within error
 ```
 
 Benchmark/ips will report the number of iterations per second for a given block
@@ -94,6 +91,18 @@ One benefit to using this method is benchmark-ips automatically determines the
 data points for testing our code, so we can focus on the results instead of
 guessing iteration counts as we do with the traditional Benchmark library.
 
+You can also use `ips_quick` to save a few lines of code:
+
+```ruby
+Benchmark.ips_quick(:upcase, :downcase, on: "hello") # runs a suite comparing "hello".upcase and "hello".downcase
+
+def first; MyJob.perform(1); end
+def second; MyJobOptimized.perform(1); end
+Benchmark.ips_quick(:first, :second) # compares :first and :second
+```
+
+This adds a very small amount of overhead, which may be significant (i.e. ips_quick will understate the difference) if you're microbenchmarking things that can do over 1 million iterations per second. In that case, you're better off using the full format.
+
 ### Custom Suite
 
 Pass a custom suite to disable garbage collection during benchmark:

data/examples/quick.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require 'benchmark/ips'
+
+def add
+  1 + 1
+end
+
+def sub
+  2 - 1
+end
+
+Benchmark.ips_quick(:add, :sub, warmup: 1, time: 1)
+
+h = {}
+
+Benchmark.ips_quick(:size, :empty?, on: h)

data/lib/benchmark/compare.rb

@@ -56,6 +56,8 @@ module Benchmark
     def compare(*entries, order: :fastest)
       return if entries.size < 2
 
+      max_width = entries.map { |e| e.label.to_s.size }.max
+
       case order
       when :baseline
         baseline = entries.shift
@@ -64,17 +66,17 @@ module Benchmark
         sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
         baseline = sorted.shift
       else
-        raise ArgumentError, "Unknwon order: #{order.inspect}"
+        raise ArgumentError, "Unknown order: #{order.inspect}"
       end
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", baseline.label.to_s, baseline.stats.central_tendency
+      $stdout.printf "%#{max_width}s: %10.1f i/s\n", baseline.label.to_s, baseline.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
 
-        $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
+        $stdout.printf "%#{max_width}s: %10.1f i/s - ", name, report.stats.central_tendency
 
         if report.stats.overlaps?(baseline.stats)
           $stdout.print "same-ish: difference falls within error"

data/lib/benchmark/ips/job/stream_report.rb

@@ -2,9 +2,10 @@ module Benchmark
   module IPS
     class Job
       class StreamReport
-        def initialize(stream = $stdout)
+        def initialize(job, stream = $stdout)
           @last_item = nil
           @out = stream
+          @job = job
         end
 
         def start_warming
@@ -17,8 +18,9 @@ module Benchmark
         end
 
         def warming(label, _warmup)
-          @out.print rjust(label)
+          @out.print label.to_s.rjust(@job.max_width)
         end
+        alias_method :running, :warming
 
         def warmup_stats(_warmup_time_us, timing)
           case format
@@ -29,8 +31,6 @@ module Benchmark
           end
         end
 
-        alias_method :running, :warming
-
         def add_report(item, caller)
           @out.puts " #{item.body}"
           @last_item = item
@@ -48,18 +48,6 @@ module Benchmark
         def format
           Benchmark::IPS.options[:format]
         end
-
-        # Add padding to label's right if label's length < 20,
-        # Otherwise add a new line and 20 whitespaces.
-        # @return [String] Right justified label.
-        def rjust(label)
-          label = label.to_s
-          if label.size > 20
-            "#{label}\n#{' ' * 20}"
-          else
-            label.rjust(20)
-          end
-        end
       end
     end
   end

data/lib/benchmark/ips/job.rb

@@ -9,7 +9,8 @@ module Benchmark
       # The percentage of the expected runtime to allow
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
-      POW_2_30 = 1 << 30
+      # Keep iterations below this to avoid overflow to 64-bit long or Bignum
+      MAX_ITERATIONS = 1 << 30
 
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
@@ -51,6 +52,10 @@ module Benchmark
       # @return [Integer]
       attr_accessor :confidence
 
+      # The maximum label width
+      # @return [Integer]
+      attr_reader :max_width
+
       # Silence output
       # @return [Boolean]
       def quiet
@@ -72,6 +77,7 @@ module Benchmark
         @compare_order = :fastest
         @held_path = nil
         @held_results = nil
+        @max_width = 20 # automatically computed as entries are added
 
         @timing = Hash.new 1 # default to 1 in case warmup isn't run
         @full_report = Report.new
@@ -85,7 +91,7 @@ module Benchmark
         @stats = :sd
         @confidence = 95
 
-        @out = MultiReport.new(StreamReport.new)
+        @out = MultiReport.new(StreamReport.new(self))
       end
 
       # Job configuration options, set +@warmup+ and +@time+.
@@ -106,7 +112,7 @@ module Benchmark
         if val # remove instances of StreamReport
           @out.quiet!
         else # ensure there is an instance of StreamReport
-          @out << StreamReport.new if @out.quiet?
+          @out << StreamReport.new(self) if @out.quiet?
         end
       end
 
@@ -180,6 +186,8 @@ module Benchmark
         action = str || blk
         raise ArgumentError, "no block or string" unless action
 
+        @max_width = label.size if label.size > @max_width
+
         @list.push Entry.new(label, action)
         self
       end
@@ -203,7 +211,7 @@ module Benchmark
         (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
       end
 
-      # Calculate the interations per second given the number
+      # Calculate the iterations per second given the number
       # of cycles run and the time in microseconds that elapsed.
       # @param [Integer] cycles Cycles.
       # @param [Integer] time_us Time in microsecond.
@@ -219,7 +227,7 @@ module Benchmark
         JSON.load(IO.read(@held_path)).each do |result|
           @held_results[result['item']] = result
           create_report(result['item'], result['measured_us'], result['iter'],
-                        create_stats(result['samples']), result['cycles'])
+                        create_stats(result['samples'], result['measured_us'], result['iter']), result['cycles'])
         end
       end
 
@@ -289,11 +297,13 @@ module Benchmark
 
             # If the number of cycles would go outside the 32-bit signed integers range
             # then exit the loop to avoid overflows and start the 100ms warmup runs
-            break if cycles >= POW_2_30
+            break if cycles >= MAX_ITERATIONS
             cycles *= 2
           end while Timing.now + warmup_time_us * 2 < target
 
-          cycles = cycles_per_100ms warmup_time_us, warmup_iter
+          per_100ms = cycles_per_100ms warmup_time_us, warmup_iter
+          # Not [per_100ms, MAX_ITERATIONS].min as that can promote the result to long
+          cycles = per_100ms > MAX_ITERATIONS ? MAX_ITERATIONS : per_100ms
           @timing[item] = cycles
 
           # Run for the remaining of warmup in a similar way as #run_benchmark.
@@ -349,7 +359,7 @@ module Benchmark
             iterations_per_sec cycles, time_us
           }
 
-          rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)
+          rep = create_report(item.label, measured_us, iter, create_stats(samples, measured_us, iter), cycles)
 
           if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
             rep.show_total_time!
@@ -361,10 +371,10 @@ module Benchmark
         end
       end
 
-      def create_stats(samples)
+      def create_stats(samples, measured_us, iterations)
         case @stats
           when :sd
-            Stats::SD.new(samples)
+            Stats::SD.new(samples, measured_us, iterations)
           when :bootstrap
             Stats::Bootstrap.new(samples, @confidence)
           else

data/lib/benchmark/ips/report.rb

@@ -62,7 +62,7 @@ module Benchmark
 
         # Control if the total time the job took is reported.
         # Typically this value is not significant because it's very
-        # close to the expected time, so it's supressed by default.
+        # close to the expected time, so it's suppressed by default.
         def show_total_time!
           @show_total_time = true
         end
@@ -86,23 +86,33 @@ module Benchmark
         # percentage of standard deviation, iterations in runtime.
         # @return [String] Left justified body.
         def body
+          per_iter = (" (%s/i)" % Helpers.humanize_duration(1_000_000_000 / @stats.central_tendency)).rjust(15)
+
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
+            central_tendency = Helpers.scale(@stats.central_tendency)
+            error_percentage = @stats.error_percentage
+            left =
+              if error_percentage > 99.9
+                "%s (± Inf%%) i/s" % [central_tendency]
+              else
+                "%s (±%4.1f%%) i/s" % [central_tendency, @stats.error_percentage]
+              end.ljust(20)
+
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
-              left.ljust(20) + (" - %s in %10.6fs" % [iters, runtime])
+              left + per_iter + (" - %s in %10.6fs" % [iters, runtime])
             else
-              left.ljust(20) + (" - %s" % iters)
+              left + per_iter + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
+            left = ("%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]).ljust(20)
 
             if @show_total_time
-              left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
+              left + per_iter + (" - %10d in %10.6fs" % [@iterations, runtime])
             else
-              left.ljust(20) + (" - %10d" % @iterations)
+              left + per_iter + (" - %10d" % @iterations)
             end
           end
         end
@@ -113,7 +123,7 @@ module Benchmark
           @label.to_s.rjust(20)
         end
 
-        # Return string repesentation of Entry object.
+        # Return string representation of Entry object.
         # @return [String] Header and body.
         def to_s
           "#{header} #{body}"

data/lib/benchmark/ips/share.rb

@@ -17,7 +17,7 @@ module Benchmark
         base = (ENV['SHARE_URL'] || DEFAULT_URL)
         url = URI(File.join(base, "reports"))
 
-        req = Net::HTTP::Post.new(url)
+        req = Net::HTTP::Post.new(url, initheader = {'Content-Type' =>'application/json'})
 
         data = {
           "entries" => @report.data,

data/lib/benchmark/ips/stats/sd.rb

@@ -6,9 +6,9 @@ module Benchmark
         include StatsMetric
         attr_reader :error, :samples
 
-        def initialize(samples)
+        def initialize(samples, measured_us, iterations)
           @samples = samples
-          @mean = Timing.mean(samples)
+          @mean = Timing::MICROSECONDS_PER_SECOND * (iterations.to_f / measured_us)
           @error = Timing.stddev(samples, @mean).round
         end
 

data/lib/benchmark/ips.rb

@@ -13,14 +13,17 @@ require 'benchmark/ips/job'
 # Performance benchmarking library
 module Benchmark
   # Benchmark in iterations per second, no more guessing!
-  # @see https://github.com/evanphx/benchmark-ips
+  #
+  # See Benchmark.ips for documentation on using this gem~
+  #
+  # @see {https://github.com/evanphx/benchmark-ips}
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.13.0"
+    VERSION = "2.15.0"
 
     # CODENAME of current version.
-    CODENAME = "Long Awaited"
+    CODENAME = "Harmonic Nice"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -73,6 +76,32 @@ module Benchmark
       report
     end
 
+    # Quickly compare multiple methods on the same object.
+    # @param methods [Symbol...] A list of method names (as symbols) to compare.
+    # @param receiver [Object] The object on which to call the methods. Defaults to Kernel.
+    # @param opts [Hash] Additional options for customizing the benchmark.
+    # @option opts [Integer] :warmup The number of seconds to warm up the benchmark.
+    # @option opts [Integer] :time The number of seconds to run the benchmark.
+    #
+    # @example Compare String#upcase and String#downcase
+    #   ips_quick(:upcase, :downcase, on: "hello")
+    #
+    # @example Compare two methods you just defined, with a custom warmup.
+    #   def add; 1+1; end
+    #   def sub; 2-1; end
+    #   ips_quick(:add, :sub, warmup: 10)
+    def ips_quick(*methods, on: Kernel, **opts)
+      ips(opts) do |x|
+        x.compare!
+
+        methods.each do |name|
+          x.report(name) do |iter|
+            iter.times { on.__send__ name }
+          end
+        end
+      end
+    end
+
     # Set options for running the benchmarks.
     # :format => [:human, :raw]
     #    :human format narrows precision and scales results for readability
@@ -83,20 +112,33 @@ module Benchmark
 
     module Helpers
       SUFFIXES = ['', 'k', 'M', 'B', 'T', 'Q'].freeze
-    
+
       def scale(value)
-        scale = (Math.log10(value) / 3).to_i 
+        scale = (Math.log10(value) / 3).to_i
         scale = 0 if scale < 0 || scale >= SUFFIXES.size
         suffix = SUFFIXES[scale]
         scaled_value = value.to_f / (1000 ** scale)
-    
+
         "%10.3f#{suffix}" % scaled_value
       end
       module_function :scale
+
+      def humanize_duration(duration_ns)
+        if duration_ns < 1000
+          "%.2f ns" % duration_ns
+        elsif duration_ns < 1_000_000
+          "%.2f μs" % (duration_ns / 1000)
+        elsif duration_ns < 1_000_000_000
+          "%.2f ms" % (duration_ns / 1_000_000)
+        else
+          "%.2f s" % (duration_ns / 1_000_000_000)
+        end
+      end
+      module_function :humanize_duration
     end
   end
 
-  extend Benchmark::IPS # make ips available as module-level method
+  extend Benchmark::IPS # make ips/ips_quick available as module-level method
 
   ##
   # :singleton-method: ips

data/lib/benchmark/timing.rb

@@ -4,9 +4,9 @@ module Benchmark
     # Microseconds per second.
     MICROSECONDS_PER_SECOND = 1_000_000
 
-    # Calculate (arithmetic) mean of given samples.
+    # Calculate arithmetic mean of given numbers.
     # @param [Array] samples Samples to calculate mean.
-    # @return [Float] Mean of given samples.
+    # @return [Float] Mean of given numbers.
     def self.mean(samples)
       sum = samples.inject(:+)
       sum / samples.size

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.13.0
+  version: 2.15.0
 platform: ruby
 authors:
 - Evan Phoenix
@@ -53,6 +53,7 @@ files:
 - README.md
 - examples/advanced.rb
 - examples/hold.rb
+- examples/quick.rb
 - examples/save.rb
 - examples/simple.rb
 - lib/benchmark/compare.rb
@@ -88,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.20
 signing_key:
 specification_version: 4
 summary: A iterations per second enhancement to Benchmark.