benchmark-ips 2.3.0 → 2.10.0

data/lib/benchmark/ips/noop_suite.rb

@@ -0,0 +1,25 @@
+module Benchmark
+  module IPS
+    class NoopSuite
+      def start_warming
+      end
+
+      def start_running
+      end
+
+      def footer
+      end
+
+      def warming(a, b)
+      end
+
+      def warmup_stats(a, b)
+      end
+
+      def add_report(a, b)
+      end
+
+      alias_method :running, :warming
+    end
+  end
+end

data/lib/benchmark/ips/report.rb

@@ -3,7 +3,7 @@
 module Benchmark
   module IPS
 
-    # Report contains benchamrking entries.
+    # Report contains benchmarking entries.
     # Perform operations like add new entry, run comparison between entries.
     class Report
 
@@ -13,15 +13,13 @@ module Benchmark
         # @param [#to_s] label Label of entry.
         # @param [Integer] us Measured time in microsecond.
         # @param [Integer] iters Iterations.
-        # @param [Float] ips Iterations per second.
-        # @param [Float] ips_sd Standard deviation of iterations per second.
+        # @param [Object] stats Statistics.
         # @param [Integer] cycles Number of Cycles.
-        def initialize(label, us, iters, ips, ips_sd, cycles)
+        def initialize(label, us, iters, stats, cycles)
           @label = label
           @microseconds = us
           @iterations = iters
-          @ips = ips
-          @ips_sd = ips_sd
+          @stats = stats
           @measurement_cycle = cycles
           @show_total_time = false
         end
@@ -38,13 +36,25 @@ module Benchmark
         # @return [Integer] number of iterations.
         attr_reader :iterations
 
-        # Iterations per second.
+        # Statistical summary of samples.
+        # @return [Object] statisical summary.
+        attr_reader :stats
+
+        # LEGACY: Iterations per second.
         # @return [Float] number of iterations per second.
-        attr_reader :ips
+        def ips
+          @stats.central_tendency
+        end
 
-        # Standard deviation of iteration per second.
+        # LEGACY: Standard deviation of iteration per second.
         # @return [Float] standard deviation of iteration per second.
-        attr_reader :ips_sd
+        def ips_sd
+          @stats.error
+        end
+
+        def samples
+          @stats.samples
+        end
 
         # Number of Cycles.
         # @return [Integer] number of cycles.
@@ -65,8 +75,8 @@ module Benchmark
 
         # Return entry's standard deviation of iteration per second in percentage.
         # @return [Float] +@ips_sd+ in percentage.
-        def stddev_percentage
-          100.0 * (@ips_sd.to_f / @ips.to_f)
+        def error_percentage
+          @stats.error_percentage
         end
 
         alias_method :runtime, :seconds
@@ -78,7 +88,7 @@ module Benchmark
         def body
           case Benchmark::IPS.options[:format]
           when :human
-            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(ips), stddev_percentage]
+            left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
             iters = Helpers.scale(@iterations)
 
             if @show_total_time
@@ -87,7 +97,7 @@ module Benchmark
               left.ljust(20) + (" - %s" % iters)
             end
           else
-            left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
+            left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
 
             if @show_total_time
               left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
@@ -117,8 +127,8 @@ module Benchmark
 
       # class Report
 
-      # Entry to represent each benchamarked code in Report.
-      # @return [Array<Entry>] Entries in Report.
+      # Entry to represent each benchmarked code in Report.
+      # @return [Array<Report::Entry>] Entries in Report.
       attr_reader :entries
 
       # Instantiate the Report.
@@ -131,13 +141,14 @@ module Benchmark
       # @param label [String] Entry label.
       # @param microseconds [Integer] Measured time in microsecond.
       # @param iters [Integer] Iterations.
-      # @param ips [Float] Average Iterations per second.
-      # @param ips_sd [Float] Standard deviation of iterations per second.
+      # @param stats [Object] Statistical results.
       # @param measurement_cycle [Integer] Number of cycles.
-      # @return [Entry] Last added entry.
-      def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
-        @entries << Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
-        @entries.last
+      # @return [Report::Entry] Last added entry.
+      def add_entry label, microseconds, iters, stats, measurement_cycle
+        entry = Entry.new(label, microseconds, iters, stats, measurement_cycle)
+        @entries.delete_if { |e| e.label == label }
+        @entries << entry
+        entry
       end
 
       # Entries data in array for generate json.
@@ -145,20 +156,28 @@ module Benchmark
       #   name:   Entry#label
       #   ips:    Entry#ips
       #   stddev: Entry#ips_sd
-      # @return [Array] Array of entries
+      #   microseconds: Entry#microseconds
+      #   iterations:   Entry#iterations
+      #   cycles:       Entry#measurement_cycles
+      # @return [Array<Hash<Symbol,String|Float|Integer>] Array of hashes
       def data
         @data ||= @entries.collect do |entry|
           {
             :name => entry.label,
-            :ips =>  entry.ips,
-            :stddev => entry.ips_sd
+            :central_tendency =>  entry.stats.central_tendency,
+            :ips =>  entry.stats.central_tendency, # for backwards compatibility
+            :error => entry.stats.error,
+            :stddev => entry.stats.error, # for backwards compatibility
+            :microseconds => entry.microseconds,
+            :iterations => entry.iterations,
+            :cycles => entry.measurement_cycle,
           }
         end
       end
 
       # Run comparison of entries.
-      def run_comparison
-        Benchmark.compare(*@entries)
+      def run_comparison(order)
+        Benchmark.compare(*@entries, order: order)
       end
 
       # Generate json from Report#data to given path.

data/lib/benchmark/ips/share.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'net/https'
+require 'json'
+
+module Benchmark
+  module IPS
+    class Share
+      DEFAULT_URL = "https://ips.fastruby.io"
+      def initialize(report, job)
+        @report = report
+        @job = job
+      end
+
+      def share
+        base = (ENV['SHARE_URL'] || DEFAULT_URL)
+        url = URI(File.join(base, "reports"))
+
+        req = Net::HTTP::Post.new(url)
+
+        data = {
+          "entries" => @report.data,
+          "options" => {
+            "compare" => @job.compare?
+          }
+        }
+
+        req.body = JSON.generate(data)
+
+        http = Net::HTTP.new(url.hostname, url.port)
+        if url.scheme == "https"
+          http.use_ssl = true
+          http.ssl_version = :TLSv1_2
+        end
+
+        res = http.start do |h|
+          h.request req
+        end
+
+        if Net::HTTPOK === res
+          data = JSON.parse res.body
+          puts "Shared at: #{File.join(base, data["id"])}"
+        else
+          puts "Error sharing report"
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/ips/stats/bootstrap.rb

@@ -0,0 +1,58 @@
+module Benchmark
+  module IPS
+    module Stats
+
+      class Bootstrap
+        include StatsMetric
+        attr_reader :data, :error, :samples
+
+        def initialize(samples, confidence)
+          dependencies
+          @iterations = 10_000
+          @confidence = (confidence / 100.0).to_s
+          @samples = samples
+          @data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
+          interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
+          @median = interval.median
+          @error = interval.error
+        end
+
+        # Average stat value
+        # @return [Float] central_tendency
+        def central_tendency
+          @median
+        end
+
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
+        def slowdown(baseline)
+          low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
+          error = Timing.mean([slowdown - low, high - slowdown])
+          [slowdown, error]
+        end
+
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
+        def footer
+          "with #{(@confidence.to_f * 100).round(1)}% confidence"
+        end
+
+        def dependencies
+          require 'kalibera'
+        rescue LoadError
+          puts
+          puts "Can't load the kalibera gem - this is required to use the :bootstrap stats options."
+          puts "It's optional, so we don't formally depend on it and it isn't installed along with benchmark-ips."
+          puts "You probably want to do something like 'gem install kalibera' to fix this."
+          abort
+        end
+
+      end
+
+    end
+  end
+end

data/lib/benchmark/ips/stats/sd.rb

@@ -0,0 +1,45 @@
+module Benchmark
+  module IPS
+    module Stats
+
+      class SD
+        include StatsMetric
+        attr_reader :error, :samples
+
+        def initialize(samples)
+          @samples = samples
+          @mean = Timing.mean(samples)
+          @error = Timing.stddev(samples, @mean).round
+        end
+
+        # Average stat value
+        # @return [Float] central_tendency
+        def central_tendency
+          @mean
+        end
+
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
+        def slowdown(baseline)
+          if baseline.central_tendency > central_tendency
+            [baseline.central_tendency.to_f / central_tendency, 0]
+          else
+            [central_tendency.to_f / baseline.central_tendency, 0]
+          end
+        end
+
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
+        def footer
+          nil
+        end
+
+      end
+
+    end
+  end
+end

data/lib/benchmark/ips/stats/stats_metric.rb

@@ -0,0 +1,21 @@
+module Benchmark
+  module IPS
+    module Stats
+      module StatsMetric
+        # Return entry's standard deviation of iteration per second in percentage.
+        # @return [Float] +@ips_sd+ in percentage.
+        def error_percentage
+          100.0 * (error.to_f / central_tendency)
+        end
+
+        def overlaps?(baseline)
+          baseline_low = baseline.central_tendency - baseline.error
+          baseline_high = baseline.central_tendency + baseline.error
+          my_high = central_tendency + error
+          my_low  = central_tendency - error
+          my_high > baseline_low && my_low < baseline_high
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/ips.rb

@@ -1,7 +1,14 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/stats_metric'
+require 'benchmark/ips/stats/sd'
+require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
+require 'benchmark/ips/noop_suite'
+require 'benchmark/ips/job/entry'
+require 'benchmark/ips/job/stdout_report'
+require 'benchmark/ips/job/noop_report'
 require 'benchmark/ips/job'
 
 # Performance benchmarking library
@@ -11,10 +18,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.3.0"
+    VERSION = "2.10.0"
 
     # CODENAME of current version.
-    CODENAME = "Monsoon BBQ"
+    CODENAME = "Watashi Wa Genki"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -28,47 +35,43 @@ module Benchmark
         time, warmup, quiet = args
       end
 
-      suite = nil
-
       sync, $stdout.sync = $stdout.sync, true
 
-      if defined? Benchmark::Suite and Suite.current
-        suite = Benchmark::Suite.current
-      end
-
-      quiet ||= (suite && suite.quiet?)
-
-      job = Job.new({:suite => suite,
-                     :quiet => quiet
-      })
+      job = Job.new
 
       job_opts = {}
       job_opts[:time] = time unless time.nil?
       job_opts[:warmup] = warmup unless warmup.nil?
+      job_opts[:quiet] = quiet unless quiet.nil?
 
       job.config job_opts
 
       yield job
 
-      $stdout.puts "Calculating -------------------------------------" unless quiet
-
-      job.run_warmup
-
-      $stdout.puts "-------------------------------------------------" unless quiet
+      job.load_held_results
 
       job.run
 
+      if job.run_single? && job.all_results_have_been_run?
+        job.clear_held_results
+      else
+        job.save_held_results
+        puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
+      end
+
       $stdout.sync = sync
+      job.run_comparison
+      job.generate_json
 
-      if job.compare?
-        job.run_comparison
-      end
+      report = job.full_report
 
-      if job.json?
-        job.generate_json
+      if ENV['SHARE'] || ENV['SHARE_URL']
+        require 'benchmark/ips/share'
+        share = Share.new report, job
+        share.share
       end
 
-      return job.full_report
+      report
     end
 
     # Set options for running the benchmarks.
@@ -100,4 +103,68 @@ module Benchmark
   end
 
   extend Benchmark::IPS # make ips available as module-level method
+
+  ##
+  # :singleton-method: ips
+  #
+  #     require 'benchmark/ips'
+  #
+  #     Benchmark.ips do |x|
+  #       # Configure the number of seconds used during
+  #       # the warmup phase (default 2) and calculation phase (default 5)
+  #       x.config(:time => 5, :warmup => 2)
+  #
+  #       # These parameters can also be configured this way
+  #       x.time = 5
+  #       x.warmup = 2
+  #
+  #       # Typical mode, runs the block as many times as it can
+  #       x.report("addition") { 1 + 2 }
+  #
+  #       # To reduce overhead, the number of iterations is passed in
+  #       # and the block must run the code the specific number of times.
+  #       # Used for when the workload is very small and any overhead
+  #       # introduces incorrectable errors.
+  #       x.report("addition2") do |times|
+  #         i = 0
+  #         while i < times
+  #           1 + 2
+  #           i += 1
+  #         end
+  #       end
+  #
+  #       # To reduce overhead even more, grafts the code given into
+  #       # the loop that performs the iterations internally to reduce
+  #       # overhead. Typically not needed, use the |times| form instead.
+  #       x.report("addition3", "1 + 2")
+  #
+  #       # Really long labels should be formatted correctly
+  #       x.report("addition-test-long-label") { 1 + 2 }
+  #
+  #       # Compare the iterations per second of the various reports!
+  #       x.compare!
+  #     end
+  #
+  # This will generate the following report:
+  #
+  #     Calculating -------------------------------------
+  #                 addition    71.254k i/100ms
+  #                addition2    68.658k i/100ms
+  #                addition3    83.079k i/100ms
+  #     addition-test-long-label
+  #                             70.129k i/100ms
+  #     -------------------------------------------------
+  #                 addition     4.955M (± 8.7%) i/s -     24.155M
+  #                addition2    24.011M (± 9.5%) i/s -    114.246M
+  #                addition3    23.958M (±10.1%) i/s -    115.064M
+  #     addition-test-long-label
+  #                              5.014M (± 9.1%) i/s -     24.545M
+  #
+  #     Comparison:
+  #                addition2: 24011974.8 i/s
+  #                addition3: 23958619.8 i/s - 1.00x slower
+  #     addition-test-long-label:  5014756.0 i/s - 4.79x slower
+  #                 addition:  4955278.9 i/s - 4.85x slower
+  #
+  # See also Benchmark::IPS
 end

data/lib/benchmark/timing.rb

@@ -1,12 +1,14 @@
 module Benchmark
   # Perform caclulations on Timing results.
   module Timing
+    # Microseconds per second.
+    MICROSECONDS_PER_SECOND = 1_000_000
 
     # Calculate (arithmetic) mean of given samples.
     # @param [Array] samples Samples to calculate mean.
     # @return [Float] Mean of given samples.
     def self.mean(samples)
-      sum = samples.inject(0) { |acc, i| acc + i }
+      sum = samples.inject(:+)
       sum / samples.size
     end
 
@@ -29,21 +31,7 @@ module Benchmark
       Math.sqrt variance(samples, m)
     end
 
-    # Resample mean of given samples.
-    # @param [Integer] resample_times Resample times, defaults to 100.
-    # @return [Array] Resampled samples.
-    def self.resample_mean(samples, resample_times=100)
-      resamples = []
-
-      resample_times.times do
-        resample = samples.map { samples[rand(samples.size)] }
-        resamples << Timing.mean(resample)
-      end
-
-      resamples
-    end
-
-    # Recycle unsed objects by starting Garbage Collector.
+    # Recycle used objects by starting Garbage Collector.
     def self.clean_env
       # rbx
       if GC.respond_to? :run
@@ -52,5 +40,40 @@ module Benchmark
         GC.start
       end
     end
+
+    # Use a monotonic clock if available, otherwise use Time
+    begin
+      Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond
+
+      # Get an object that represents now and can be converted to microseconds
+      def self.now
+        Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond
+      end
+
+      # Add one second to the time represenetation
+      def self.add_second(t, s)
+        t + (s * MICROSECONDS_PER_SECOND)
+      end
+
+      # Return the number of microseconds between the 2 moments
+      def self.time_us(before, after)
+        after - before
+      end
+    rescue NameError
+      # Get an object that represents now and can be converted to microseconds
+      def self.now
+        Time.now
+      end
+
+      # Add one second to the time represenetation
+      def self.add_second(t, s)
+        t + s
+      end
+
+      # Return the number of microseconds between the 2 moments
+      def self.time_us(before, after)
+        (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
+      end
+    end
   end
 end