benchmark-ips 2.8.0 → 2.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 54899cb6088746f3b585d3e471795fa63c5fa06b9a452e7923d5211ea096eff1
-  data.tar.gz: 21f3a97a2971d77dbf33fa05a497f7c61499497da66b8aff26987c64d6e2de3c
+  metadata.gz: e6c664c1e49fd7235ba573dc4a5bd92ce7446c0d4206c355a2c7b84320b26d5a
+  data.tar.gz: 72d1502418ff11b5e321825ad5d3d1c44889d170c7da0becbf8b6b9fd7c5f283
 SHA512:
-  metadata.gz: 3b7dd9271181dc58b32864d0d60ecf58c8daf92dc024ab764bc07289a39650db30354d72fc8f586207c6f5ccfaccf72c41eb9ac75d4e7be7af7a18c60f6e7860
-  data.tar.gz: 11731f19a4fbc1abcc6e31ad6c5d4ea752f26d3a5ce9573b4e07011b568c455abfbd5a53b5e41408d7e931a5327d1c6c0a2200c25d10897b9d4111d45326c7d5
+  metadata.gz: '0956333ead3cb5307864b8d013828fff4f8dbec88c27142314d2742ba5378dbf0388f04706bf797cb24d2fd21a38576cdf04892d62664eb418d6cba38e5bfb33'
+  data.tar.gz: 0a953ca0e9270f4eac9c00c51117bc2bfcba6a06e51f3508152b3dcbfd8e03ae4e0cebbf79bc6dc51efcf200e459d9a5ee457d70f371f946e1da8a75c50f7462

data/History.txt

@@ -1,3 +1,43 @@
+=== 2.9.1 / 2021-05-24
+
+* Bug fix
+  * Include all files in gem
+
+=== 2.9.0 / 2021-05-21
+
+* Features
+  * Suite can now be set via an accessor
+  * Default SHARE_URL is now `ips.fastruby.io`, operated by Ombu Labs.
+
+=== 2.8.4 / 2020-12-03
+
+* Bug fix
+  * Fixed hold! when results file does not exist.
+
+=== 2.8.3 / 2020-08-28
+
+* Bug fix
+  * Fixed inaccuracy caused by integer overflows.
+
+=== 2.8.2 / 2020-05-04
+
+* Bug fix
+  * Fixed problems with Manifest.txt.
+  * Empty interim results files are ignored.
+
+=== 2.8.0 / 2020-05-01
+
+* Feature
+  * Allow running with empty ips block.
+  * Added save! method for saving interim results.
+  * Run more than just 1 cycle during warmup to reduce overhead.
+  * Optimized Job::Entry hot-path for fairer results on JRuby/TruffleRuby.
+
+* Bug fix
+  * Removed the warmup section if set to 0.
+  * Added some RDoc docs.
+  * Added some examples in examples/
+
 === 2.7.2 / 2016-08-18
 
 * 1 bug fix:

data/Manifest.txt

@@ -7,10 +7,13 @@ lib/benchmark/compare.rb
 lib/benchmark/ips.rb
 lib/benchmark/ips/job.rb
 lib/benchmark/ips/job/entry.rb
+lib/benchmark/ips/job/noop_report.rb
 lib/benchmark/ips/job/stdout_report.rb
+lib/benchmark/ips/noop_suite.rb
 lib/benchmark/ips/report.rb
 lib/benchmark/ips/share.rb
 lib/benchmark/ips/stats/bootstrap.rb
 lib/benchmark/ips/stats/sd.rb
+lib/benchmark/ips/stats/stats_metric.rb
 lib/benchmark/timing.rb
 test/test_benchmark_ips.rb

data/README.md

@@ -186,11 +186,15 @@ end
 
 ### Online sharing
 
-If you want to share quickly your benchmark result with others. Run you benchmark
-with `SHARE=1` argument. I.e.: `SHARE=1 ruby my_benchmark.rb`.
-Result will be sent to [benchmark.fyi](https://benchmark.fyi/) and benchmark-ips
+If you want to quickly share your benchmark result with others, run you benchmark
+with `SHARE=1` argument. For example: `SHARE=1 ruby my_benchmark.rb`.
+
+Result will be sent to [benchmark.fyi](https://ips.fastruby.io/) and benchmark-ips
 will display the link to share the benchmark's result.
 
+If you want to run your own instance of [benchmark.fyi](https://github.com/evanphx/benchmark.fyi)
+and share it to that instance, you can do this: `SHARE_URL=https://ips.example.com ruby my_benchmark.rb`
+
 ### Advanced Statistics
 
 By default, the margin of error shown is plus-minus one standard deviation. If
@@ -227,7 +231,7 @@ Benchmark.ips do |x|
 
   x.stats = :bootstrap
   x.confidence = 95
-  
+
   # confidence is 95% by default, so it can be omitted
 
 end

data/lib/benchmark/ips.rb

@@ -5,8 +5,10 @@ require 'benchmark/ips/stats/stats_metric'
 require 'benchmark/ips/stats/sd'
 require 'benchmark/ips/stats/bootstrap'
 require 'benchmark/ips/report'
+require 'benchmark/ips/noop_suite'
 require 'benchmark/ips/job/entry'
 require 'benchmark/ips/job/stdout_report'
+require 'benchmark/ips/job/noop_report'
 require 'benchmark/ips/job'
 
 # Performance benchmarking library
@@ -16,10 +18,10 @@ module Benchmark
   module IPS
 
     # Benchmark-ips Gem version.
-    VERSION = "2.8.0"
+    VERSION = "2.9.1"
 
     # CODENAME of current version.
-    CODENAME = "Tardy Turtle"
+    CODENAME = "Sleepy Sasquatch"
 
     # Measure code in block, each code's benchmarked result will display in
     # iteration per second with standard deviation in given time.
@@ -33,23 +35,14 @@ module Benchmark
         time, warmup, quiet = args
       end
 
-      suite = nil
-
       sync, $stdout.sync = $stdout.sync, true
 
-      if defined? Benchmark::Suite and Suite.current
-        suite = Benchmark::Suite.current
-      end
-
-      quiet ||= (suite && suite.quiet?)
-
-      job = Job.new({:suite => suite,
-                     :quiet => quiet
-      })
+      job = Job.new
 
       job_opts = {}
       job_opts[:time] = time unless time.nil?
       job_opts[:warmup] = warmup unless warmup.nil?
+      job_opts[:quiet] = quiet unless quiet.nil?
 
       job.config job_opts
 
@@ -109,6 +102,8 @@ module Benchmark
     end
   end
 
+  extend Benchmark::IPS # make ips available as module-level method
+
   ##
   # :singleton-method: ips
   #
@@ -172,5 +167,4 @@ module Benchmark
   #                 addition:  4955278.9 i/s - 4.85x slower
   #
   # See also Benchmark::IPS
-  extend Benchmark::IPS # make ips available as module-level method
 end

data/lib/benchmark/ips/job.rb

@@ -9,6 +9,7 @@ module Benchmark
       # The percentage of the expected runtime to allow
       # before reporting a weird runtime
       MAX_TIME_SKEW = 0.05
+      POW_2_30 = 1 << 30
 
       # Two-element arrays, consisting of label and block pairs.
       # @return [Array<Entry>] list of entries
@@ -50,16 +51,20 @@ module Benchmark
       # @return [Integer]
       attr_accessor :confidence
 
+      # Silence output
+      # @return [Boolean]
+      attr_reader :quiet
+
+      # Suite
+      # @return [Benchmark::IPS::NoopSuite]
+      attr_reader :suite
+
       # Instantiate the Benchmark::IPS::Job.
-      # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
-      # @option opts [Boolean] (false) :quiet Suppress the printing of information.
       def initialize opts={}
-        @suite = opts[:suite] || nil
-        @stdout = opts[:quiet] ? nil : StdoutReport.new
         @list = []
-        @compare = false
         @run_single = false
         @json_path = false
+        @compare = false
         @held_path = nil
         @held_results = nil
 
@@ -87,6 +92,20 @@ module Benchmark
         @iterations = opts[:iterations] if opts[:iterations]
         @stats = opts[:stats] if opts[:stats]
         @confidence = opts[:confidence] if opts[:confidence]
+        self.quiet = opts[:quiet]
+        self.suite = opts[:suite]
+      end
+
+      def quiet=(val)
+        @stdout = reporter(quiet: val)
+      end
+
+      def suite=(suite)
+        @suite = suite || Benchmark::IPS::NoopSuite.new
+      end
+
+      def reporter(quiet:)
+        quiet ? NoopReport.new : StdoutReport.new
       end
 
       # Return true if job needs to be compared.
@@ -187,7 +206,7 @@ module Benchmark
       end
 
       def load_held_results
-        return unless @held_path && File.exist?(@held_path)
+        return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
         require "json"
         @held_results = {}
         JSON.load(IO.read(@held_path)).each do |result|
@@ -222,19 +241,19 @@ module Benchmark
 
       def run
         if @warmup && @warmup != 0 then
-          @stdout.start_warming if @stdout
+          @stdout.start_warming
           @iterations.times do
             run_warmup
           end
         end
 
-        @stdout.start_running if @stdout
+        @stdout.start_running
 
         @iterations.times do |n|
           run_benchmark
         end
 
-        @stdout.footer if @stdout
+        @stdout.footer
       end
 
       # Run warmup.
@@ -242,8 +261,8 @@ module Benchmark
         @list.each do |item|
           next if run_single? && @held_results && @held_results.key?(item.label)
 
-          @suite.warming item.label, @warmup if @suite
-          @stdout.warming item.label, @warmup if @stdout
+          @suite.warming item.label, @warmup
+          @stdout.warming item.label, @warmup
 
           Timing.clean_env
 
@@ -263,6 +282,10 @@ module Benchmark
             t1 = Timing.now
             warmup_iter = cycles
             warmup_time_us = Timing.time_us(t0, t1)
+
+            # If the number of cycles would go outside the 32-bit signed integers range
+            # then exit the loop to avoid overflows and start the 100ms warmup runs
+            break if cycles >= POW_2_30
             cycles *= 2
           end
 
@@ -275,8 +298,8 @@ module Benchmark
             item.call_times cycles
           end
 
-          @stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
-          @suite.warmup_stats warmup_time_us, @timing[item] if @suite
+          @stdout.warmup_stats warmup_time_us, @timing[item]
+          @suite.warmup_stats warmup_time_us, @timing[item]
 
           break if run_single?
         end
@@ -287,8 +310,8 @@ module Benchmark
         @list.each do |item|
           next if run_single? && @held_results && @held_results.key?(item.label)
 
-          @suite.running item.label, @time if @suite
-          @stdout.running item.label, @time if @stdout
+          @suite.running item.label, @time
+          @stdout.running item.label, @time
 
           Timing.clean_env
 
@@ -329,8 +352,8 @@ module Benchmark
             rep.show_total_time!
           end
 
-          @stdout.add_report rep, caller(1).first if @stdout
-          @suite.add_report rep, caller(1).first if @suite
+          @stdout.add_report rep, caller(1).first
+          @suite.add_report rep, caller(1).first
 
           break if run_single?
         end

data/lib/benchmark/ips/job/entry.rb

@@ -11,10 +11,12 @@ module Benchmark
         def initialize(label, action)
           @label = label
 
+          # We define #call_times on the singleton class of each Entry instance.
+          # That way, there is no polymorphism for `@action.call` inside #call_times.
+
           if action.kind_of? String
-            compile action
+            compile_string action
             @action = self
-            @as_action = true
           else
             unless action.respond_to? :call
               raise ArgumentError, "invalid action, must respond to #call"
@@ -23,12 +25,10 @@ module Benchmark
             @action = action
 
             if action.respond_to? :arity and action.arity > 0
-              @call_loop = true
+              compile_block_with_manual_loop
             else
-              @call_loop = false
+              compile_block
             end
-
-            @as_action = false
           end
         end
 
@@ -40,25 +40,43 @@ module Benchmark
         # @return [String, Proc] Code to be called, could be String / Proc.
         attr_reader :action
 
-        # Call action by given times, return if +@call_loop+ is present.
+        # Call action by given times.
         # @param times [Integer] Times to call +@action+.
         # @return [Integer] Number of times the +@action+ has been called.
         def call_times(times)
-          return @action.call(times) if @call_loop
+          raise '#call_times should be redefined per Benchmark::IPS::Job::Entry instance'
+        end
 
-          act = @action
+        def compile_block
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              act = @action
 
-          i = 0
-          while i < times
-            act.call
-            i += 1
-          end
+              i = 0
+              while i < times
+                act.call
+                i += 1
+              end
+            end
+          CODE
+          m.class_eval code
+        end
+
+        def compile_block_with_manual_loop
+          m = (class << self; self; end)
+          code = <<-CODE
+            def call_times(times)
+              @action.call(times)
+            end
+          CODE
+          m.class_eval code
         end
 
         # Compile code into +call_times+ method.
         # @param str [String] Code to be compiled.
         # @return [Symbol] :call_times.
-        def compile(str)
+        def compile_string(str)
           m = (class << self; self; end)
           code = <<-CODE
             def call_times(__total);

data/lib/benchmark/ips/job/noop_report.rb

@@ -0,0 +1,27 @@
+module Benchmark
+  module IPS
+    class Job
+      class NoopReport
+        def start_warming
+        end
+
+        def start_running
+        end
+
+        def footer
+        end
+
+        def warming(a, b)
+        end
+
+        def warmup_stats(a, b)
+        end
+
+        def add_report(a, b)
+        end
+
+        alias_method :running, :warming
+      end
+    end
+  end
+end

data/lib/benchmark/ips/noop_suite.rb

@@ -0,0 +1,25 @@
+module Benchmark
+  module IPS
+    class NoopSuite
+      def start_warming
+      end
+
+      def start_running
+      end
+
+      def footer
+      end
+
+      def warming(a, b)
+      end
+
+      def warmup_stats(a, b)
+      end
+
+      def add_report(a, b)
+      end
+
+      alias_method :running, :warming
+    end
+  end
+end

data/lib/benchmark/ips/share.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'net/https'
 require 'json'
@@ -5,7 +7,7 @@ require 'json'
 module Benchmark
   module IPS
     class Share
-      DEFAULT_URL = "https://benchmark.fyi"
+      DEFAULT_URL = "https://ips.fastruby.io"
       def initialize(report, job)
         @report = report
         @job = job

data/lib/benchmark/ips/stats/stats_metric.rb

@@ -0,0 +1,21 @@
+module Benchmark
+  module IPS
+    module Stats
+      module StatsMetric
+        # Return entry's standard deviation of iteration per second in percentage.
+        # @return [Float] +@ips_sd+ in percentage.
+        def error_percentage
+          100.0 * (error.to_f / central_tendency)
+        end
+
+        def overlaps?(baseline)
+          baseline_low = baseline.central_tendency - baseline.error
+          baseline_high = baseline.central_tendency + baseline.error
+          my_high = central_tendency + error
+          my_low  = central_tendency - error
+          my_high > baseline_low && my_low < baseline_high
+        end
+      end
+    end
+  end
+end

data/test/test_benchmark_ips.rb

@@ -1,6 +1,7 @@
 require "minitest/autorun"
 require "benchmark/ips"
 require "stringio"
+require "tmpdir"
 
 class TestBenchmarkIPS < Minitest::Test
   def setup
@@ -53,6 +54,13 @@ class TestBenchmarkIPS < Minitest::Test
     end
 
     assert $stdout.string.size.zero?
+
+    Benchmark.ips do |x|
+      x.quiet = true
+      x.report("operation") { 100 * 100 }
+    end
+
+    assert $stdout.string.size.zero?
   end
 
   def test_ips
@@ -116,6 +124,21 @@ class TestBenchmarkIPS < Minitest::Test
     assert_equal [:warming, :warmup_stats, :running, :add_report], suite.calls
   end
 
+  def test_ips_config_suite_by_accsr
+    suite = Struct.new(:calls) do
+      def method_missing(method, *args)
+        calls << method
+      end
+    end.new([])
+
+    Benchmark.ips(0.1, 0.1) do |x|
+      x.suite = suite
+      x.report("job") {}
+    end
+
+    assert_equal [:warming, :warmup_stats, :running, :add_report], suite.calls
+  end
+
   def test_ips_defaults
     report = Benchmark.ips do |x|
       x.report("sleep 0.25") { sleep(0.25) }
@@ -182,4 +205,17 @@ class TestBenchmarkIPS < Minitest::Test
     assert data[0]["ips"]
     assert data[0]["stddev"]
   end
+
+  def test_hold!
+    temp_file_name = Dir::Tmpname.create(["benchmark-ips", ".tmp"]) { }
+
+    Benchmark.ips(:time => 0.001, :warmup => 0.001) do |x|
+      x.report("operation") { 100 * 100 }
+      x.report("operation2") { 100 * 100 }
+      x.hold! temp_file_name
+    end
+
+    assert File.exist?(temp_file_name)
+    File.unlink(temp_file_name)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark-ips
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 2.9.1
 platform: ruby
 authors:
 - Evan Phoenix
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-01 00:00:00.000000000 Z
+date: 2021-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -77,11 +77,14 @@ files:
 - lib/benchmark/ips.rb
 - lib/benchmark/ips/job.rb
 - lib/benchmark/ips/job/entry.rb
+- lib/benchmark/ips/job/noop_report.rb
 - lib/benchmark/ips/job/stdout_report.rb
+- lib/benchmark/ips/noop_suite.rb
 - lib/benchmark/ips/report.rb
 - lib/benchmark/ips/share.rb
 - lib/benchmark/ips/stats/bootstrap.rb
 - lib/benchmark/ips/stats/sd.rb
+- lib/benchmark/ips/stats/stats_metric.rb
 - lib/benchmark/timing.rb
 - test/test_benchmark_ips.rb
 homepage: https://github.com/evanphx/benchmark-ips
@@ -106,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.9
 signing_key: 
 specification_version: 4
 summary: An iterations per second enhancement to Benchmark.