benchmark-ips 2.3.0 → 2.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ac378dafa55fa25f223374c352d9fce265a73a1d
-  data.tar.gz: edcbad0a3ed40b69793e18b52b40b4d2d73b72fd
+SHA256:
+  metadata.gz: 194a6da5977a23dc733ade6dadaefe1a7d7215678d6135ff33d5c37304bec993
+  data.tar.gz: e072ffd46009a79e13990e7435e1415ec24fd0407f86d566208fc597b8e33f7d
 SHA512:
-  metadata.gz: 427e884910dc92a0e50ceb825ff36770b70c0dcee9088f56d3e090867986f6c8d12d176c7a671d7b964f15c12c2ae6edd4c87b532c6c68bdd351c0508c03fd90
-  data.tar.gz: 3db5f1f4445eb0b2c4985633a197abf58be95f9a2bb01282bda329c2880916e1c25c5cd0ec94b2a17adbdc667e3e8829dd46fa428ef13dc41cf3327136564d89
+  metadata.gz: 5b3a8e41f223d3945daf11329e7cd68b1cc41f9b08b9f3f2e9e5a471a37134782f4e1f78caca102b7b3b08eb179ac4d78868bb1f97bfab4ac1ccef22aa38d42d
+  data.tar.gz: e5cd27c03050929b8edf402264c0f5a32560550ee6d79456d2513ae9b4a76f763f669a32ba4cc3bf7a45e7c086426631c2ba26724a749192fc637046bfd6a495

data/History.md

@@ -0,0 +1,228 @@
+### 2.11.0 / 2023-02-15
+
+* Feature
+  * Adds .json! method to the ips block argument, allowing you to print the output as JSON to a file or STDOUT.
+
+### 2.10.0 / 2022-02-17
+
+* Feature
+  * Adds :order option to compare, with new `:baseline` order which compares all
+    variations against the first option benchmarked.
+
+### 2.9.3 / 2022-01-25
+
+* Bug fix
+  * All warmups and benchmarks must run at least once
+
+### 2.9.2 / 2021-10-10
+
+* Bug fix
+  * Fix a problem with certain configs of quiet mode
+
+### 2.9.1 / 2021-05-24
+
+* Bug fix
+  * Include all files in gem
+
+### 2.9.0 / 2021-05-21
+
+* Features
+  * Suite can now be set via an accessor
+  * Default SHARE_URL is now `ips.fastruby.io`, operated by Ombu Labs.
+
+### 2.8.4 / 2020-12-03
+
+* Bug fix
+  * Fixed hold! when results file does not exist.
+
+### 2.8.3 / 2020-08-28
+
+* Bug fix
+  * Fixed inaccuracy caused by integer overflows.
+
+### 2.8.2 / 2020-05-04
+
+* Bug fix
+  * Fixed problems with Manifest.txt.
+  * Empty interim results files are ignored.
+
+### 2.8.0 / 2020-05-01
+
+* Feature
+  * Allow running with empty ips block.
+  * Added save! method for saving interim results.
+  * Run more than just 1 cycle during warmup to reduce overhead.
+  * Optimized Job::Entry hot-path for fairer results on JRuby/TruffleRuby.
+
+* Bug fix
+  * Removed the warmup section if set to 0.
+  * Added some RDoc docs.
+  * Added some examples in examples/
+
+### 2.7.2 / 2016-08-18
+
+* 1 bug fix:
+  * Restore old accessors. Fixes #76
+
+### 2.7.1 / 2016-08-08
+
+Add missing files
+
+### 2.7.0 / 2016-08-05
+
+* 1 minor features:
+  * Add support for confidence intervals
+
+* 1 bug fixes:
+  * Cleanup a few coding patterns
+
+* 2 doc fixes:
+  * Add infos about benchark.fyi to Readme
+  * Remove ancient releases
+
+* 3 merged PRs:
+  * Merge pull request #65 from kbrock/fixup_inject
+  * Merge pull request #67 from benoittgt/master
+  * Merge pull request #69 from chrisseaton/kalibera-confidence-intervals
+
+### MISSING 2.6.0 and 2.6.1
+
+### 2.5.0 / 2016-02-14
+
+* 1 minor feature:
+  * Add iterations option.
+
+* 1 bug fixes:
+  * Don't tell people something is slower if it's within the error.
+
+* 2 merged PRs:
+  * Merge pull request #58 from chrisseaton/iterations
+  * Merge pull request #60 from chrisseaton/significance
+
+### 2.4.1 / 2016-02-12
+
+* 1 bug fix:
+  * Add missing files to gem
+
+### 2.4.0 / 2016-02-12
+
+* 1 minor features
+  * Add support for hold! and independent invocations.
+
+* 6 bug fixes
+  * Separate messages for warming up and calculating.
+  * Tighten timing loop.
+  * Pass simple types into Job#create_report
+  * More concise sorting
+  * Fix runtime comparison
+  * Use runtime if ips is not available
+
+* 5 doc fixes
+  * Fix typo unsed --> used
+  * Better document Report::Entry
+  * Fix some typos in docs
+  * Don't calculate mean 2 times
+  * Add more tolerance to tests
+
+* 13 merged PRs
+  * Merge pull request #44 from kbrock/job_extract
+  * Merge pull request #45 from kbrock/runtime_only
+  * Merge pull request #47 from kbrock/use_avg
+  * Merge pull request #46 from kbrock/report_stdout
+  * Merge pull request #48 from bquorning/fix-label-for-runtime-comparison
+  * Merge pull request #50 from tjschuck/fix_typo
+  * Merge pull request #51 from bquorning/all-reports-respond-to-ips
+  * Merge pull request #52 from kbrock/document_reports
+  * Merge pull request #53 from kbrock/interface_create_report
+  * Merge pull request #54 from PragTob/patch-2
+  * Merge pull request #55 from chrisseaton/messages
+  * Merge pull request #56 from chrisseaton/independence
+  * Merge pull request #57 from chrisseaton/tighten-loop
+
+### 2.3.0 / 2015-07-20
+
+* 2 minor features:
+  * Support keyword arguments
+  * Allow any datatype for labels (use #to_s conversion)
+
+* 1 doc/test changes:
+  * Newer Travis for 1.8.7, ree, and 2.2.2
+
+* 3 PRs merged:
+  * Merge pull request #41 from kbrock/kwargs-support
+  * Merge pull request #42 from kbrock/newer_travis
+  * Merge pull request #43 from kbrock/non_to_s_labels
+
+### 2.2.0 / 2015-05-09
+
+* 1 minor features:
+  * Fix quiet mode
+  * Allow passing a custom suite via config
+  * Silent a job if a suite was passed and is quiet
+  * Export report to json file.
+  * Accept symbol as report's argument.
+
+* 2 doc fixes:
+  * Squish duplicate `to` in README
+  * Update copyright to 2015. [ci skip]
+
+* 9 PRs merged:
+  * Merge pull request #37 from splattael/patch-1
+  * Merge pull request #36 from kirs/quiet-mode
+  * Merge pull request #35 from JuanitoFatas/doc/suite
+  * Merge pull request #34 from splattael/config-suite
+  * Merge pull request #33 from splattael/suite-quiet
+  * Merge pull request #32 from O-I/remove-gemfile-lock
+  * Merge pull request #31 from JuanitoFatas/doc/bump-copyright-year
+  * Merge pull request #29 from JuanitoFatas/feature/json-export
+  * Merge pull request #26 from JuanitoFatas/feature/takes-symbol-as-report-parameter
+
+### 2.1.1 / 2015-01-12
+
+* 1 minor fix:
+  * Don't send label through printf so that % work directly
+
+* 1 documenation changes:
+  * Use HEREDOC and wrap at 80 chars for example result description
+
+* 1 usage fix:
+  * Add gemspec for use via bundler git
+
+* 1 PR merged:
+  * Merge pull request #24 from zzak/simple-format-result-description
+
+### 2.1.0 / 2014-11-10
+
+* Documentation changes:
+  * Many documentation fixes by Juanito Fatas!
+  * Minor readme fix by Will Leinweber
+
+* 2 minor features:
+  * Displaying the total runtime for a job is suppressed unless interesting
+  * Formatting of large values improved (human vs raw mode)
+    * Contributed by Charles Oliver Nutter
+
+### 2.0.0 / 2014-06-18
+
+* The 'Davy Stevenson' release!
+  * Codename: Springtime Hummingbird Dance
+
+ * Big API refactoring so the internal bits are easier to use
+ * Bump to 2.0 because return types changed to make the API better
+
+* Contributors added:
+  *  Davy Stevenson
+  *  Juanito Fatas
+  *  Benoit Daloze
+  *  Matias
+  *  Tony Arcieri
+  *  Vipul A M
+  *  Zachary Scott
+  *  schneems (Richard Schneeman)
+
+### 1.0.0 / 2012-03-23
+
+* 1 major enhancement
+
+  * Birthday!
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 Evan Phoenix
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,16 +1,17 @@
+# benchmark-ips
+
+* rdoc :: http://rubydoc.info/gems/benchmark-ips
+* home :: https://github.com/evanphx/benchmark-ips
+
 [![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
 [![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
 [![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
 
-# benchmark-ips
-
 * https://github.com/evanphx/benchmark-ips
 
-* [documentation](http://rubydoc.info/gems/benchmark-ips)
-
 ## DESCRIPTION:
 
-A iterations per second enhancement to Benchmark.
+An iterations per second enhancement to Benchmark.
 
 ## FEATURES/PROBLEMS:
 
@@ -136,6 +137,129 @@ Benchmark.ips do |x|
 end
 ```
 
+### Independent benchmarking
+
+If you are comparing multiple implementations of a piece of code you may want
+to benchmark them in separate invocations of Ruby so that the measurements
+are independent of each other. You can do this with the `hold!` command.
+
+```ruby
+Benchmark.ips do |x|
+
+  # Hold results between multiple invocations of Ruby
+  x.hold! 'filename'
+
+end
+```
+
+This will run only one benchmarks each time you run the command, storing
+results in the specified file. The file is deleted when all results have been
+gathered and the report is shown.
+
+Alternatively, if you prefer a different approach, the `save!` command is
+available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
+the `examples/` directory.
+
+
+### Multiple iterations
+
+In some cases you may want to run multiple iterations of the warmup and
+calculation stages and take only the last result for comparison. This is useful
+if you are benchmarking with an implementation of Ruby that optimizes using
+tracing or on-stack-replacement, because to those implementations the
+calculation phase may appear as new, unoptimized code.
+
+You can do this with the `iterations` option, which by default is `1`. The
+total time spent will then be `iterations * warmup + iterations * time` seconds.
+
+```ruby
+Benchmark.ips do |x|
+
+  x.config(:iterations => 3)
+
+    # or
+
+  x.iterations = 3
+
+end
+```
+
+### Online sharing
+
+If you want to quickly share your benchmark result with others, run you benchmark
+with `SHARE=1` argument. For example: `SHARE=1 ruby my_benchmark.rb`.
+
+Result will be sent to [benchmark.fyi](https://ips.fastruby.io/) and benchmark-ips
+will display the link to share the benchmark's result.
+
+If you want to run your own instance of [benchmark.fyi](https://github.com/evanphx/benchmark.fyi)
+and share it to that instance, you can do this: `SHARE_URL=https://ips.example.com ruby my_benchmark.rb`
+
+### Advanced Statistics
+
+By default, the margin of error shown is plus-minus one standard deviation. If
+a more advanced statistical test is wanted, a bootstrap confidence interval
+can be calculated instead. A bootstrap confidence interval has the advantages of
+arguably being more mathematically sound for this application than a standard
+deviation, it additionally produces an error for relative slowdowns, which the
+standard deviation does not, and it is arguably more intuitive and actionable.
+
+When a bootstrap confidence interval is used, a median of the interval is used
+rather than the mean of the samples, which is what you get with the default
+standard deviation.
+
+The bootstrap confidence interval used is the one described by Tomas Kalibera.
+Note that for this technique to be valid your benchmark should have reached a
+non-periodic steady state with statistically independent samples (it should
+have warmed up) by the time measurements start.
+
+Using a bootstrap confidence internal requires that the 'kalibera' gem is
+installed separately. This gem is not a formal dependency, as by default it is
+not needed.
+
+```
+gem install kalibera
+```
+
+```ruby
+Benchmark.ips do |x|
+
+  # The default is :stats => :sd, which doesn't have a configurable confidence
+  x.config(:stats => :bootstrap, :confidence => 95)
+
+    # or
+
+  x.stats = :bootstrap
+  x.confidence = 95
+
+  # confidence is 95% by default, so it can be omitted
+
+end
+```
+
+### Output as JSON
+
+You can generate output in JSON. If you want to write JSON to a file, pass filename to `json!` method:
+
+```ruby
+Benchmark.ips do |x|
+  x.report("some report") {  }
+  x.json! 'filename.json'
+end
+```
+
+If you want to write JSON to STDOUT, pass `STDOUT` to `json!` method and set `quiet = true` before `json!`:
+
+```ruby
+Benchmark.ips do |x|
+  x.report("some report") {  }
+  x.quiet = true
+  x.json! STDOUT
+end
+```
+
+This is useful when the output from `benchmark-ips` becomes an input of other tools via stdin.
+
 ## REQUIREMENTS:
 
 * None!
@@ -153,27 +277,3 @@ After checking out the source, run:
 This task will install any missing dependencies, run the tests/specs,
 and generate the RDoc.
 
-## LICENSE:
-
-(The MIT License)
-
-Copyright (c) 2015 Evan Phoenix
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/examples/advanced.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+
+  # Use bootstrap confidence intervals
+  x.stats = :bootstrap
+
+  # Set confidence to 95%
+  x.confidence = 95
+
+  # Run multiple iterations for better warmup
+  x.iterations = 3
+
+  x.report("mul") { 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 }
+  x.report("pow") { 2 ** 8 }
+
+  x.compare!
+end

data/examples/hold.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+# example to explain hold! usage https://github.com/evanphx/benchmark-ips/issues/85
+# The hold! feature expects to be run twice, generally with different Rubys.
+# hold! can also be used to compare modules changes which impact the run time
+# RUN_1: ruby examples/hold.rb
+# Warming up --------------------------------------
+#             without   172.168k i/100ms
+# Calculating -------------------------------------
+#             without      2.656M (± 3.3%) i/s -     13.429M in   5.062098s
+#
+# RUN_2: WITH_MODULE=true ruby examples/hold.rb
+# Warming up --------------------------------------
+#                 with    92.087k i/100ms
+# Calculating -------------------------------------
+#                 with      1.158M (± 1.4%) i/s -      5.801M in   5.010084s
+#
+# Comparison:
+#              without:  2464721.3 i/s
+#                 with:  1158179.6 i/s - 2.13x  slower
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+  x.report('without') do
+    'Bruce'.inspect
+  end
+
+  if ENV['WITH_MODULE'] == 'true'
+    class String
+      def inspect
+        result = %w[Bruce Wayne is Batman]
+        result.join(' ')
+      end
+    end
+  end
+
+  x.report('with') do
+    'Bruce'.inspect
+  end
+  x.hold! 'temp_results'
+  x.compare!
+end

data/examples/save.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# example to explain save!
+# The save! feature expects to be run twice, generally with different Rubys.
+# save! can also be used to compare modules changes which impact the run time
+#
+# If you're comparing ruby versions, Just use the version in the label
+#
+#     x.report("ruby #{RUBY_VERSION}") { 'Bruce'.inspect }
+#
+# Or use a hash
+# 
+#     x.report("version" => RUBY_VERSION, "method" => 'bruce') { 'Bruce'.inspect }
+#
+# RUN_1: SAVE_FILE='run1.out' ruby examples/hold.rb
+# Warming up --------------------------------------
+#             without   172.168k i/100ms
+# Calculating -------------------------------------
+#             without      2.656M (± 3.3%) i/s -     13.429M in   5.062098s
+#
+# RUN_2: SAVE_FILE='run1.out' WITH_MODULE=true ruby examples/hold.rb
+# Warming up --------------------------------------
+#                 with    92.087k i/100ms
+# Calculating -------------------------------------
+#                 with      1.158M (± 1.4%) i/s -      5.801M in   5.010084s
+#
+# Comparison:
+#              without:  2464721.3 i/s
+#                 with:  1158179.6 i/s - 2.13x  slower
+# CLEANUP: rm run1.out
+
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+  x.report(ENV['WITH_MODULE'] == 'true' ? 'with' : 'without') do
+    'Bruce'.inspect
+  end
+
+  if ENV['WITH_MODULE'] == 'true'
+    class String
+      def inspect
+        result = %w[Bruce Wayne is Batman]
+        result.join(' ')
+      end
+    end
+  end
+
+  x.save! ENV['SAVE_FILE'] if ENV['SAVE_FILE']
+  x.compare!
+end

data/examples/simple.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+
+  # Configure the number of seconds used during
+  # the warmup phase and calculation phase
+  x.config(:time => 5, :warmup => 2)
+
+  # These parameters can also be configured this way
+  x.time = 5
+  x.warmup = 2
+
+  # Typical mode, runs the block as many times as it can
+  x.report("addition") { 1 + 2 }
+
+  # To reduce overhead, the number of iterations is passed in
+  # and the block must run the code the specific number of times.
+  # Used for when the workload is very small and any overhead
+  # introduces incorrectable errors.
+  x.report(:addition2) do |times|
+    i = 0
+    while i < times
+      1 + 2
+      i += 1
+    end
+  end
+
+  # To reduce overhead even more, grafts the code given into
+  # the loop that performs the iterations internally to reduce
+  # overhead. Typically not needed, use the |times| form instead.
+  x.report("addition3", "1 + 2")
+
+  # Really long labels should be formatted correctly
+  x.report("addition-test-long-label") { 1 + 2 }
+
+  x.compare!
+end
+
+puts <<-EOD
+Typical results will show addition2 & addition3 to be the most performant, and
+they should perform reasonably similarly. You should see addition and
+addition-test-long-label to perform very similarly to each other (as they are
+running the same test, just with different labels), and they should both run in
+the neighborhood of 3.5 times slower than addition2 and addition3."
+EOD