benchmark-sweet 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '058103ce3e7d6b2022ecee07bd379f3bb19020d711cb1c5676509957d30b07a1'
+  data.tar.gz: 0b66514a2d1ac2918a8980027452b72158363717a01cf293aa3ba69b55737956
+SHA512:
+  metadata.gz: d44471dd676ea056f594fc8b50881eaa3dce9d7fa166f620b7bd52d0f7c3e653ed666f854e0ac4f128f64bc97bc587149c5b4e1e4bb98656cdfdbb1cbeb47fa0
+  data.tar.gz: 1e79bac462dc7c3a796666d01509fc7dadda089c14002e217557a4282751fff15795da14f5f0b5c1680971c7678910b449062fcf29f36109859ffd10986d7d1b

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/examples/*.json
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.4.4
+before_install: gem install bundler -v 1.16.6

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Change Log
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/)
+and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Fixed
+
+## [0.2.0] - 2020-05-11
+
+### Added
+
+- changelog
+- docs
+
+### Fixed
+
+- better color support
+- updated benchmark-ips and no longer need to monkey patch for it.
+- require benchmark-sweet now works
+- documentation fixes (thanks @d-m-u)
+- example fixes
+
+
+## 0.0.1 - 2014-05-31
+### Added
+- good stuff
+
+[Unreleased]: https://github.com/kbrock/benchmark-sweet/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/kbrock/benchmark-sweet/compare/v0.0.1...v0.2.0

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in benchmark-sweet.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Keenan Brock
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,246 @@
+# Benchmark::Sweet
+
+Time tends not to be consistent across multiple runs. But numbers of queries or the number of objects allocated tend to be more similar.
+
+This gem allows the user to collect all three of these benchmarks using a single framework similar to the benchmark and benchmark-ips syntax.
+
+Sometimes a benchmark needs to be collected across multiple runs using different gem versions or using different ruby versions. This can be done as well.
+
+Lastly, this allows multiple axes of comparisons to be performed. Example: instead of measuring multiple split implementations, it allows measuring these implementations using empty strings and long strings oo a bigger picture can be obtained.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'benchmark-sweet'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install benchmark-sweet
+
+## Example 1
+
+### Code
+
+```ruby
+require "benchmark/sweet"
+require "active_support/all"
+require "more_core_extensions/all" # [].tabelize
+
+NSTRING = nil
+DELIMITER='/'.freeze
+STRING="ab/cd/ef/gh".freeze
+
+Benchmark.items(metrics: %w(ips memsize)) do |x|
+  x.metadata data: "nil" do
+    x.report("to_s.split") { NSTRING.to_s.split(DELIMITER)           }
+    x.report("?split:[]")  { NSTRING ? NSTRING.split(DELIMITER) : [] }
+  end
+  x.metadata data: "str" do
+    x.report("to_s.split") { STRING.to_s.split(DELIMITER)          }
+    x.report("?split:[]")  { STRING ? STRING.split(DELIMITER) : [] }
+  end
+
+  # partition the data by data value (nil vs string)
+  # that way we're not comparing a split on a nil vs a split on a populated string
+  compare_by :data
+
+  # each row is a different method (via `row: :method`)
+  # each column is by data type (via `column: :data` - specified via `metadata data: "nil"`)
+  x.report_with grouping: :metric, sort: true, row: :method, column: :data
+end
+```
+
+#### `compare_by`
+
+The code takes a different amount of time to process a `nil` vs a string. So the values
+are given metadata with the appropriate data used (i.e.: `metadata data: "nil"`).
+The benchmark is then told that the results need to be partitioned by `data` (i.e.: `compare_by :data`).
+
+The multipliers are only comparing values with the same data value and do not compare `"string"` values with `"nil"` values.
+
+Values for method labels (e.g.: `"to_s.split"`) and metrics (e.g.: `ips`) are already part of the partition.
+
+#### `grouping`
+
+In this example, each metric is considered distinct so each metric is given a
+unique table (i.e.: `grouping: :metric`)
+
+Metrics of `ips` and `memsize` are calculated (i.e.: `metrics: %w(ips memsize)`)
+
+#### `row`
+
+A different method is given per row (i.e. `row: :method`)
+
+#### `column`
+
+The other axis for the columns is the type of data passed (i.e.: `column: :data`)
+This is not a native value, it is specified when the items are specified (e.g.:`metadata data: "nil"`)
+
+### example output
+
+#### metric ips
+
+ method     | nil                   | str
+------------|-----------------------|---------------
+`?split:[]` | 7090539.2 i/s         | 1322010.9 i/s
+`to_s.split`| 3703981.6 i/s - 1.91x | 1311153.9 i/s
+
+#### metric memsize
+
+ method     | nil                | str
+------------|--------------------|-------------
+`?split:[]` | 40.0 bytes         | 360.0 bytes
+`to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
+
+
+## Example 2
+
+### Code
+
+```ruby
+require "benchmark/sweet"
+require "active_support/all"
+require "more_core_extensions/all" # [].tabelize
+
+NSTRING = nil
+DELIMITER='/'.freeze
+STRING="ab/cd/ef/gh".freeze
+
+Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 1, quiet: false, force: ENV["FORCE"] == "true") do |x|
+  x.metadata version: RUBY_VERSION
+  x.metadata data: "nil" do
+    x.report("to_s.split") { NSTRING.to_s.split(DELIMITER)           }
+    x.report("?split:[]")  { NSTRING ? NSTRING.split(DELIMITER) : [] }
+  end
+  x.metadata data: "str" do
+    x.report("to_s.split") { STRING.to_s.split(DELIMITER)          }
+    x.report("?split:[]")  { STRING ? STRING.split(DELIMITER) : [] }
+  end
+
+  # partition the data by ruby version and data present
+  # that way we're not comparing a split on a nil vs a split on a populated string
+  x.compare_by :version, :data
+if ENV["CONDENSED"].to_s == "true"
+  x.report_with grouping: :version, sort: true, row: :method, column: [:data, :metric]
+else
+  x.report_with grouping: [:version, :metric], sort: true, row: :method, column: :data
+end
+
+  x.save_file $PROGRAM_NAME.sub(/\.rb$/, '.json')
+end
+```
+
+#### `save_file`
+
+Creates a json save file which saves the timings across multiple runs.
+This is used along with the `version` metadata to record different results per ruby version.
+Another common use is to record ActiveRecord version or a gem's version.
+
+This is run with two different versions of ruby.
+Interim values are stored in the save_file.
+
+Running with environment variable `FORCE` will force running this again. (i.e.: `force: ENV["force"] == true`)
+
+Depending upon the environment variable `CONDENSED`, there are two types of output.
+
+#### `compare_by`
+
+We introduce `version` as metadata for the tests. Adding `version` to the comparison
+says that we should only compare values for the same version of ruby (along with the same data).
+
+If you note `version` and `data` are not reserved words, instead, they are just what metadata we
+decided to pass in.
+
+### Example 2 output
+
+#### [:version, :metric] 2.3.7_ips
+
+ method     | nil                   | str
+------------|-----------------------|---------------
+ `?split:[]`| 10146134.2 i/s        | 1284159.2 i/s
+`to_s.split`| 4232772.3 i/s - 2.40x | 1258665.8 i/s
+
+#### [:version, :metric] 2.3.7_memsize
+
+ method     | nil                | str
+------------|--------------------|-------------
+ `?split:[]`| 40.0 bytes         | 360.0 bytes
+`to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
+
+#### [:version, :metric] 2.4.6_ips
+
+ method     | nil                   | str
+------------|-----------------------|---------------
+ `?split:[]`| 10012873.4 i/s        | 1377320.5 i/s
+`to_s.split`| 4557456.3 i/s - 2.20x | 1350562.6 i/s
+
+#### [:version, :metric] 2.4.6_memsize
+
+ method     | nil                | str
+------------|--------------------|-------------
+ `?split:[]`| 40.0 bytes         | 360.0 bytes
+`to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
+
+#### [:version, :metric] 2.5.5_ips
+
+ method     | nil                   | str
+------------|-----------------------|---------------
+ `?split:[]`| 7168109.1 i/s         | 1357046.0 i/s
+`to_s.split`| 3779969.3 i/s - 1.90x | 1328072.4 i/s
+
+#### [:version, :metric] 2.5.5_memsize
+
+ method     | nil                | str
+------------|--------------------|-------------
+ `?split:[]`| 40.0 bytes         | 360.0 bytes
+`to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
+
+
+### Example 2 CONDENSED output
+
+running with `CONDENSED=true` calls with a different `report_with`
+
+As you might notice, the number of objects created for the runs are the same.
+But for some reason, the `nil` split case is slower for ruby 2.5.5.
+
+#### version 2.3.7
+
+ method     | nil_ips               | str_ips       | nil_memsize        | str_memsize
+------------|-----------------------|---------------|--------------------|-------------
+ `?split:[]`| 10146134.2 i/s        | 1284159.2 i/s | 40.0 bytes         | 360.0 bytes
+`to_s.split`| 4232772.3 i/s - 2.40x | 1258665.8 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+
+#### version 2.4.6
+
+ method     | nil_ips               | str_ips       | nil_memsize        | str_memsize
+------------|-----------------------|---------------|--------------------|-------------
+ `?split:[]`| 10012873.4 i/s        | 1377320.5 i/s | 40.0 bytes         | 360.0 bytes
+`to_s.split`| 4557456.3 i/s - 2.20x | 1350562.6 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+
+#### version 2.5.5
+
+ method     | nil_ips               | str_ips       | nil_memsize        | str_memsize
+------------|-----------------------|---------------|--------------------|-------------
+ `?split:[]`| 7168109.1 i/s         | 1357046.0 i/s | 40.0 bytes         | 360.0 bytes
+`to_s.split`| 3779969.3 i/s - 1.90x | 1328072.4 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kbrock/benchmark-sweet.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/benchmark-sweet.gemspec

@@ -0,0 +1,48 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "benchmark/sweet/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "benchmark-sweet"
+  spec.version       = Benchmark::Sweet::VERSION
+  spec.authors       = ["Keenan Brock"]
+  spec.email         = ["keenan@thebrocks.net"]
+
+  spec.summary       = %q{Suite to run multiple benchmarks}
+  spec.description   = <<-EOF
+  Benchmark Sweet is a suite to run multiple kinds of metrics.
+  It can be configured to run memory, sql query, and ips benchmarks on a common set of code.
+  This data can be collected across multiple runs of the code, to support multiple ruby or
+  gem versions.
+  This also generates more complex comparisons
+EOF
+
+  spec.homepage      = "https://github.com/kbrock/benchmark-sweet"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "http://github.com/kbrock/benchmark-sweet"
+    spec.metadata["changelog_uri"] = "http://github.com/kbrock/benchmark-sweet/CHANGELOG.md"
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.1.4"
+  spec.add_development_dependency "rake", ">= 12.3.3"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "activerecord"
+
+  spec.add_runtime_dependency "benchmark-ips", "~> 2.8.2"
+  spec.add_runtime_dependency "memory_profiler", "~> 0.9.0"
+  spec.add_runtime_dependency "more_core_extensions" # for [].tabelize        - want to remove
+  spec.add_runtime_dependency "activesupport"         # for {}.symbolize_keys! - want to remove
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "benchmark/sweet"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/benchmark_big_small.rb

@@ -0,0 +1,84 @@
+require "benchmark/sweet"
+require "active_support/all"
+require "more_core_extensions/all" # [].tabelize
+
+# output:
+#
+# [:version] 2.3.7
+#
+#  method     | nil_ips               | str_ips       | nil_memsize        | str_memsize
+# ------------+-----------------------+---------------+--------------------+-------------
+#  ?split:[]  | 51574246.6 i/s        | 1434943.3 i/s | 40.0 bytes         | 360.0 bytes
+#  to_s.split | 5707766.1 i/s - 9.04x | 1411708.0 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+#
+# [:version] 2.4.4
+#
+#  method     | nil_ips               | str_ips       | nil_memsize        | str_memsize
+# ------------+-----------------------+---------------+--------------------+-------------
+#  ?split:[]  | 51740193.6 i/s        | 1411882.2 i/s | 40.0 bytes         | 360.0 bytes
+#  to_s.split | 5887968.5 i/s - 8.79x | 1347822.1 i/s | 80.0 bytes - 2.00x | 360.0 bytes
+#
+#
+#
+# (CONDENSED=true) output:
+#
+# [:metric, :version] ips_2.3.7
+#
+#  method      | nil                    | str
+# -------------+------------------------+---------------
+#  ?split:[]   | 51825798.8 i/s         | 1407946.4 i/s
+#  &&split||[] | 46730725.8 i/s - 1.11x | 1413355.3 i/s
+#  to_s.split  | 5685237.1 i/s - 9.12x  | 1396494.3 i/s
+#
+# [:metric, :version] ips_2.4.4
+#
+#  method      | nil                    | str
+# -------------+------------------------+---------------
+#  ?split:[]   | 51559454.4 i/s         | 1438780.8 i/s
+#  &.split||[] | 46446196.0 i/s - 1.11x | 1437665.3 i/s
+#  &&split||[] | 43356335.6 i/s - 1.19x | 1434466.6 i/s
+#  to_s.split  | 5835694.7 i/s - 8.84x  | 1427819.0 i/s
+#
+# [:metric, :version] memsize_2.3.7
+#
+#  method      | nil                | str
+# -------------+--------------------+-------------
+#  ?split:[]   | 40.0 bytes         | 360.0 bytes
+#  &&split||[] | 40.0 bytes         | 360.0 bytes
+#  to_s.split  | 80.0 bytes - 2.00x | 360.0 bytes
+#
+# [:metric, :version] memsize_2.4.4
+#
+#  method      | nil                | str
+# -------------+--------------------+-------------
+#  ?split:[]   | 40.0 bytes         | 360.0 bytes
+#  &&split||[] | 40.0 bytes         | 360.0 bytes
+#  &.split||[] | 40.0 bytes         | 360.0 bytes
+#  to_s.split  | 80.0 bytes - 2.00x | 360.0 bytes
+
+NSTRING = nil
+DELIMITER='/'.freeze
+STRING="ab/cd/ef/gh".freeze
+
+Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 1, quiet: false, force: ENV["FORCE"] == "true") do |x|
+  x.metadata version: RUBY_VERSION
+  x.metadata data: "nil" do
+    x.report("to_s.split",  "NSTRING.to_s.split(DELIMITER)")
+    x.report("?split:[]",   "NSTRING ? NSTRING.split(DELIMITER) : []")
+  end
+  x.metadata data: :str do
+    x.report("to_s.split",  "STRING.to_s.split(DELIMITER)")
+    x.report("?split:[]",   "STRING ? STRING.split(DELIMITER) : []")
+  end
+
+  # partition the data by ruby version and data present
+  # that way we're not comparing a split on a nil vs a split on a populated string
+  x.compare_by :version, :data
+if ENV["CONDENSED"].to_s == "true"
+  x.report_with grouping: :version, sort: true, row: :method, column: [:data, :metric]
+else
+  x.report_with grouping: [:version, :metric], sort: true, row: :method, column: :data
+end
+
+  x.save_file (ENV["SAVE_FILE"] == "true") ? $0.sub(/\.rb$/, '.json') : ENV["SAVE_FILE"] if ENV["SAVE_FILE"]
+end