benchmark-ips 2.3.0 → 2.11.0

data/lib/benchmark/ips/share.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'net/https'
+require 'json'
+
+module Benchmark
+  module IPS
+    class Share
+      DEFAULT_URL = "https://ips.fastruby.io"
+      def initialize(report, job)
+        @report = report
+        @job = job
+      end
+
+      def share
+        base = (ENV['SHARE_URL'] || DEFAULT_URL)
+        url = URI(File.join(base, "reports"))
+
+        req = Net::HTTP::Post.new(url)
+
+        data = {
+          "entries" => @report.data,
+          "options" => {
+            "compare" => @job.compare?
+          }
+        }
+
+        req.body = JSON.generate(data)
+
+        http = Net::HTTP.new(url.hostname, url.port)
+        if url.scheme == "https"
+          http.use_ssl = true
+          http.ssl_version = :TLSv1_2
+        end
+
+        res = http.start do |h|
+          h.request req
+        end
+
+        if Net::HTTPOK === res
+          data = JSON.parse res.body
+          puts "Shared at: #{File.join(base, data["id"])}"
+        else
+          puts "Error sharing report"
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/ips/stats/bootstrap.rb

@@ -0,0 +1,58 @@
+module Benchmark
+  module IPS
+    module Stats
+
+      class Bootstrap
+        include StatsMetric
+        attr_reader :data, :error, :samples
+
+        def initialize(samples, confidence)
+          dependencies
+          @iterations = 10_000
+          @confidence = (confidence / 100.0).to_s
+          @samples = samples
+          @data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
+          interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
+          @median = interval.median
+          @error = interval.error
+        end
+
+        # Average stat value
+        # @return [Float] central_tendency
+        def central_tendency
+          @median
+        end
+
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
+        def slowdown(baseline)
+          low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
+          error = Timing.mean([slowdown - low, high - slowdown])
+          [slowdown, error]
+        end
+
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
+        def footer
+          "with #{(@confidence.to_f * 100).round(1)}% confidence"
+        end
+
+        def dependencies
+          require 'kalibera'
+        rescue LoadError
+          puts
+          puts "Can't load the kalibera gem - this is required to use the :bootstrap stats options."
+          puts "It's optional, so we don't formally depend on it and it isn't installed along with benchmark-ips."
+          puts "You probably want to do something like 'gem install kalibera' to fix this."
+          abort
+        end
+
+      end
+
+    end
+  end
+end

data/lib/benchmark/ips/stats/sd.rb

@@ -0,0 +1,45 @@
+module Benchmark
+  module IPS
+    module Stats
+
+      class SD
+        include StatsMetric
+        attr_reader :error, :samples
+
+        def initialize(samples)
+          @samples = samples
+          @mean = Timing.mean(samples)
+          @error = Timing.stddev(samples, @mean).round
+        end
+
+        # Average stat value
+        # @return [Float] central_tendency
+        def central_tendency
+          @mean
+        end
+
+        # Determines how much slower this stat is than the baseline stat
+        # if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
+        # @param baseline [SD|Bootstrap] faster baseline
+        # @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
+        def slowdown(baseline)
+          if baseline.central_tendency > central_tendency
+            [baseline.central_tendency.to_f / central_tendency, nil]
+          else
+            [central_tendency.to_f / baseline.central_tendency, nil]
+          end
+        end
+
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
+        def footer
+          nil
+        end
+
+      end
+
+    end
+  end
+end

data/lib/benchmark/ips/stats/stats_metric.rb

@@ -0,0 +1,21 @@
+module Benchmark
+  module IPS
+    module Stats
+      module StatsMetric
+        # Return entry's standard deviation of iteration per second in percentage.
+        # @return [Float] +@ips_sd+ in percentage.
+        def error_percentage
+          100.0 * (error.to_f / central_tendency)
+        end
+
+        def overlaps?(baseline)
+          baseline_low = baseline.central_tendency - baseline.error
+          baseline_high = baseline.central_tendency + baseline.error
+          my_high = central_tendency + error
+          my_low  = central_tendency - error
+          my_high > baseline_low && my_low < baseline_high
+        end
+      end
+    end
+  end
+end

data/lib/benchmark/ips.rb

@@ -1,7 +1,14 @@
 # encoding: utf-8
 require 'benchmark/timing'
 require 'benchmark/compare'
+require 'benchmark/ips/stats/stats_metric'
+require 'benchmark/ips/stats/sd'
+require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
+require 'benchmark/ips/noop_suite'
+require 'benchmark/ips/job/entry'
+require 'benchmark/ips/job/stdout_report'
+require 'benchmark/ips/job/noop_report'
 require 'benchmark/ips/job'
 
 # Performance benchmarking library
@@ -11,10 +18,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.3.0"
+    VERSION = "2.11.0"
 
     # CODENAME of current version.
-    CODENAME = "Monsoon BBQ"
+    CODENAME = "We Do This Once A Year"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -28,47 +35,43 @@ module Benchmark
         time, warmup, quiet = args
       end
 
-      suite = nil
-
       sync, $stdout.sync = $stdout.sync, true
 
-      if defined? Benchmark::Suite and Suite.current
-        suite = Benchmark::Suite.current
-      end
-
-      quiet ||= (suite && suite.quiet?)
-
-      job = Job.new({:suite => suite,
-                     :quiet => quiet
-      })
+      job = Job.new
 
       job_opts = {}
       job_opts[:time] = time unless time.nil?
       job_opts[:warmup] = warmup unless warmup.nil?
+      job_opts[:quiet] = quiet unless quiet.nil?
 
       job.config job_opts
 
       yield job
 
-      $stdout.puts "Calculating -------------------------------------" unless quiet
-
-      job.run_warmup
-
-      $stdout.puts "-------------------------------------------------" unless quiet
+      job.load_held_results
 
       job.run
 
+      if job.run_single? && job.all_results_have_been_run?
+        job.clear_held_results
+      else
+        job.save_held_results
+        puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
+      end
+
       $stdout.sync = sync
+      job.run_comparison
+      job.generate_json
 
-      if job.compare?
-        job.run_comparison
-      end
+      report = job.full_report
 
-      if job.json?
-        job.generate_json
+      if ENV['SHARE'] || ENV['SHARE_URL']
+        require 'benchmark/ips/share'
+        share = Share.new report, job
+        share.share
       end
 
-      return job.full_report
+      report
     end
 
     # Set options for running the benchmarks.
@@ -100,4 +103,68 @@ module Benchmark
   end
 
   extend Benchmark::IPS # make ips available as module-level method
+
+  ##
+  # :singleton-method: ips
+  #
+  #     require 'benchmark/ips'
+  #
+  #     Benchmark.ips do |x|
+  #       # Configure the number of seconds used during
+  #       # the warmup phase (default 2) and calculation phase (default 5)
+  #       x.config(:time => 5, :warmup => 2)
+  #
+  #       # These parameters can also be configured this way
+  #       x.time = 5
+  #       x.warmup = 2
+  #
+  #       # Typical mode, runs the block as many times as it can
+  #       x.report("addition") { 1 + 2 }
+  #
+  #       # To reduce overhead, the number of iterations is passed in
+  #       # and the block must run the code the specific number of times.
+  #       # Used for when the workload is very small and any overhead
+  #       # introduces incorrectable errors.
+  #       x.report("addition2") do |times|
+  #         i = 0
+  #         while i < times
+  #           1 + 2
+  #           i += 1
+  #         end
+  #       end
+  #
+  #       # To reduce overhead even more, grafts the code given into
+  #       # the loop that performs the iterations internally to reduce
+  #       # overhead. Typically not needed, use the |times| form instead.
+  #       x.report("addition3", "1 + 2")
+  #
+  #       # Really long labels should be formatted correctly
+  #       x.report("addition-test-long-label") { 1 + 2 }
+  #
+  #       # Compare the iterations per second of the various reports!
+  #       x.compare!
+  #     end
+  #
+  # This will generate the following report:
+  #
+  #     Calculating -------------------------------------
+  #                 addition    71.254k i/100ms
+  #                addition2    68.658k i/100ms
+  #                addition3    83.079k i/100ms
+  #     addition-test-long-label
+  #                             70.129k i/100ms
+  #     -------------------------------------------------
+  #                 addition     4.955M (± 8.7%) i/s -     24.155M
+  #                addition2    24.011M (± 9.5%) i/s -    114.246M
+  #                addition3    23.958M (±10.1%) i/s -    115.064M
+  #     addition-test-long-label
+  #                              5.014M (± 9.1%) i/s -     24.545M
+  #
+  #     Comparison:
+  #                addition2: 24011974.8 i/s
+  #                addition3: 23958619.8 i/s - 1.00x slower
+  #     addition-test-long-label:  5014756.0 i/s - 4.79x slower
+  #                 addition:  4955278.9 i/s - 4.85x slower
+  #
+  # See also Benchmark::IPS
 end

data/lib/benchmark/timing.rb

@@ -1,12 +1,14 @@
 module Benchmark
   # Perform caclulations on Timing results.
   module Timing
+    # Microseconds per second.
+    MICROSECONDS_PER_SECOND = 1_000_000
 
     # Calculate (arithmetic) mean of given samples.
     # @param [Array] samples Samples to calculate mean.
     # @return [Float] Mean of given samples.
     def self.mean(samples)
-      sum = samples.inject(0) { |acc, i| acc + i }
+      sum = samples.inject(:+)
       sum / samples.size
     end
 
@@ -29,21 +31,7 @@ module Benchmark
       Math.sqrt variance(samples, m)
     end
 
-    # Resample mean of given samples.
-    # @param [Integer] resample_times Resample times, defaults to 100.
-    # @return [Array] Resampled samples.
-    def self.resample_mean(samples, resample_times=100)
-      resamples = []
-
-      resample_times.times do
-        resample = samples.map { samples[rand(samples.size)] }
-        resamples << Timing.mean(resample)
-      end
-
-      resamples
-    end
-
-    # Recycle unsed objects by starting Garbage Collector.
+    # Recycle used objects by starting Garbage Collector.
     def self.clean_env
       # rbx
       if GC.respond_to? :run
@@ -52,5 +40,40 @@ module Benchmark
         GC.start
       end
     end
+
+    # Use a monotonic clock if available, otherwise use Time
+    begin
+      Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond
+
+      # Get an object that represents now and can be converted to microseconds
+      def self.now
+        Process.clock_gettime Process::CLOCK_MONOTONIC, :float_microsecond
+      end
+
+      # Add one second to the time represenetation
+      def self.add_second(t, s)
+        t + (s * MICROSECONDS_PER_SECOND)
+      end
+
+      # Return the number of microseconds between the 2 moments
+      def self.time_us(before, after)
+        after - before
+      end
+    rescue NameError
+      # Get an object that represents now and can be converted to microseconds
+      def self.now
+        Time.now
+      end
+
+      # Add one second to the time represenetation
+      def self.add_second(t, s)
+        t + s
+      end
+
+      # Return the number of microseconds between the 2 moments
+      def self.time_us(before, after)
+        (after.to_f - before.to_f) * MICROSECONDS_PER_SECOND
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.11.0
 platform: ruby
 authors:
 - Evan Phoenix
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2015-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
+        version: '5.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
+        version: '5.4'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -38,47 +38,41 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: hoe
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.13'
 description: A iterations per second enhancement to Benchmark.
 email:
 - evan@phx.io
 executables: []
 extensions: []
 extra_rdoc_files:
-- History.txt
-- Manifest.txt
+- History.md
+- LICENSE
 - README.md
 files:
-- ".autotest"
-- ".gemtest"
-- History.txt
-- Manifest.txt
+- History.md
+- LICENSE
 - README.md
-- Rakefile
+- examples/advanced.rb
+- examples/hold.rb
+- examples/save.rb
+- examples/simple.rb
 - lib/benchmark/compare.rb
 - lib/benchmark/ips.rb
 - lib/benchmark/ips/job.rb
+- lib/benchmark/ips/job/entry.rb
+- lib/benchmark/ips/job/noop_report.rb
+- lib/benchmark/ips/job/stdout_report.rb
+- lib/benchmark/ips/noop_suite.rb
 - lib/benchmark/ips/report.rb
+- lib/benchmark/ips/share.rb
+- lib/benchmark/ips/stats/bootstrap.rb
+- lib/benchmark/ips/stats/sd.rb
+- lib/benchmark/ips/stats/stats_metric.rb
 - lib/benchmark/timing.rb
-- test/test_benchmark_ips.rb
 homepage: https://github.com/evanphx/benchmark-ips
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -95,9 +89,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.3.20
+signing_key:
 specification_version: 4
 summary: A iterations per second enhancement to Benchmark.
 test_files: []

data/.autotest

@@ -1,23 +0,0 @@
-# -*- ruby -*-
-
-require 'autotest/restart'
-
-# Autotest.add_hook :initialize do |at|
-#   at.extra_files << "../some/external/dependency.rb"
-#
-#   at.libs << ":../some/external"
-#
-#   at.add_exception 'vendor'
-#
-#   at.add_mapping(/dependency.rb/) do |f, _|
-#     at.files_matching(/test_.*rb$/)
-#   end
-#
-#   %w(TestA TestB).each do |klass|
-#     at.extra_class_map[klass] = "test/test_misc.rb"
-#   end
-# end
-
-# Autotest.add_hook :run_command do |at|
-#   system "rake build"
-# end

data/History.txt

@@ -1,87 +0,0 @@
-=== 2.3.0 / 2015-07-20
-
-* 2 minor features:
-  * Support keyword arguments
-  * Allow any datatype for labels (use #to_s conversion)
-
-* 1 doc/test changes:
-  * Newer Travis for 1.8.7, ree, and 2.2.2
-
-* 3 PRs merged:
-  * Merge pull request #41 from kbrock/kwargs-support
-  * Merge pull request #42 from kbrock/newer_travis
-  * Merge pull request #43 from kbrock/non_to_s_labels
-
-=== 2.2.0 / 2015-05-09
-
-* 1 minor features:
-  * Fix quiet mode
-  * Allow passing a custom suite via config
-  * Silent a job if a suite was passed and is quiet
-  * Export report to json file.
-  * Accept symbol as report's argument.
-
-* 2 doc fixes:
-  * Squish duplicate `to` in README
-  * Update copyright to 2015. [ci skip]
-
-* 9 PRs merged:
-  * Merge pull request #37 from splattael/patch-1
-  * Merge pull request #36 from kirs/quiet-mode
-  * Merge pull request #35 from JuanitoFatas/doc/suite
-  * Merge pull request #34 from splattael/config-suite
-  * Merge pull request #33 from splattael/suite-quiet
-  * Merge pull request #32 from O-I/remove-gemfile-lock
-  * Merge pull request #31 from JuanitoFatas/doc/bump-copyright-year
-  * Merge pull request #29 from JuanitoFatas/feature/json-export
-  * Merge pull request #26 from JuanitoFatas/feature/takes-symbol-as-report-parameter
-
-=== 2.1.1 / 2015-01-12
-
-* 1 minor fix:
-  * Don't send label through printf so that % work directly
-
-* 1 documenation changes:
-  * Use HEREDOC and wrap at 80 chars for example result description
-
-* 1 usage fix:
-  * Add gemspec for use via bundler git
-
-* 1 PR merged:
-  * Merge pull request #24 from zzak/simple-format-result-description
-
-=== 2.1.0 / 2014-11-10
-
-* Documentation changes:
-  * Many documentation fixes by Juanito Fatas!
-  * Minor readme fix by Will Leinweber
-
-* 2 minor features:
-  * Displaying the total runtime for a job is suppressed unless interesting
-  * Formatting of large values improved (human vs raw mode)
-    * Contributed by Charles Oliver Nutter
-
-=== 2.0.0 / 2014-06-18
-
-* The 'Davy Stevenson' release!
-  * Codename: Springtime Hummingbird Dance
-
- * Big API refactoring so the internal bits are easier to use
- * Bump to 2.0 because return types changed to make the API better
-
-* Contributors added:
-  *  Davy Stevenson
-  *  Juanito Fatas
-  *  Benoit Daloze
-  *  Matias
-  *  Tony Arcieri
-  *  Vipul A M
-  *  Zachary Scott
-  *  schneems (Richard Schneeman)
-
-=== 1.0.0 / 2012-03-23
-
-* 1 major enhancement
-
-  * Birthday!
-

data/Manifest.txt

@@ -1,11 +0,0 @@
-.autotest
-History.txt
-Manifest.txt
-README.md
-Rakefile
-lib/benchmark/compare.rb
-lib/benchmark/ips.rb
-lib/benchmark/ips/report.rb
-lib/benchmark/ips/job.rb
-lib/benchmark/timing.rb
-test/test_benchmark_ips.rb

data/Rakefile

@@ -1,26 +0,0 @@
-# -*- ruby -*-
-
-require 'rubygems'
-require 'hoe'
-
-Hoe.plugin :minitest
-Hoe.plugin :git
-
-hoe = Hoe.spec 'benchmark-ips' do
-  developer('Evan Phoenix', 'evan@phx.io')
-
-  self.readme_file = 'README.md'
-
-  license "MIT"
-end
-
-file "#{hoe.spec.name}.gemspec" => ['Rakefile', "lib/benchmark/ips.rb"] do |t|
-  puts "Generating #{t.name}"
-  File.open(t.name, 'wb') { |f| f.write hoe.spec.to_ruby }
-end
-
-desc "Generate or update the standalone gemspec file for the project"
-task :gemspec => ["#{hoe.spec.name}.gemspec"]
-
-
-# vim: syntax=ruby