benchmark-sweet 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: '058103ce3e7d6b2022ecee07bd379f3bb19020d711cb1c5676509957d30b07a1'
4
+ data.tar.gz: 0b66514a2d1ac2918a8980027452b72158363717a01cf293aa3ba69b55737956
5
+ SHA512:
6
+ metadata.gz: d44471dd676ea056f594fc8b50881eaa3dce9d7fa166f620b7bd52d0f7c3e653ed666f854e0ac4f128f64bc97bc587149c5b4e1e4bb98656cdfdbb1cbeb47fa0
7
+ data.tar.gz: 1e79bac462dc7c3a796666d01509fc7dadda089c14002e217557a4282751fff15795da14f5f0b5c1680971c7678910b449062fcf29f36109859ffd10986d7d1b
@@ -0,0 +1,12 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+ /examples/*.json
10
+
11
+ # rspec failure tracking
12
+ .rspec_status
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.4.4
7
+ before_install: gem install bundler -v 1.16.6
@@ -0,0 +1,36 @@
1
+ # Change Log
2
+ All notable changes to this project will be documented in this file.
3
+
4
+ The format is based on [Keep a Changelog](http://keepachangelog.com/)
5
+ and this project adheres to [Semantic Versioning](http://semver.org/).
6
+
7
+ ## [Unreleased]
8
+
9
+ ### Added
10
+
11
+ ### Changed
12
+
13
+ ### Fixed
14
+
15
+ ## [0.2.0] - 2020-05-11
16
+
17
+ ### Added
18
+
19
+ - changelog
20
+ - docs
21
+
22
+ ### Fixed
23
+
24
+ - better color support
25
+ - updated benchmark-ips and no longer need to monkey patch for it.
26
+ - require benchmark-sweet now works
27
+ - documentation fixes (thanks @d-m-u)
28
+ - example fixes
29
+
30
+
31
+ ## 0.0.1 - 2014-05-31
32
+ ### Added
33
+ - good stuff
34
+
35
+ [Unreleased]: https://github.com/kbrock/benchmark-sweet/compare/v0.2.0...HEAD
36
+ [0.2.0]: https://github.com/kbrock/benchmark-sweet/compare/v0.0.1...v0.2.0
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
4
+
5
+ # Specify your gem's dependencies in benchmark-sweet.gemspec
6
+ gemspec
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2018 Keenan Brock
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,246 @@
1
+ # Benchmark::Sweet
2
+
3
+ Time tends not to be consistent across multiple runs. But numbers of queries or the number of objects allocated tend to be more similar.
4
+
5
+ This gem allows the user to collect all three of these benchmarks using a single framework similar to the benchmark and benchmark-ips syntax.
6
+
7
+ Sometimes a benchmark needs to be collected across multiple runs using different gem versions or using different ruby versions. This can be done as well.
8
+
9
+ Lastly, this allows multiple axes of comparisons to be performed. Example: instead of measuring multiple split implementations, it allows measuring these implementations using empty strings and long strings oo a bigger picture can be obtained.
10
+
11
+ ## Installation
12
+
13
+ Add this line to your application's Gemfile:
14
+
15
+ ```ruby
16
+ gem 'benchmark-sweet'
17
+ ```
18
+
19
+ And then execute:
20
+
21
+ $ bundle
22
+
23
+ Or install it yourself as:
24
+
25
+ $ gem install benchmark-sweet
26
+
27
+ ## Example 1
28
+
29
+ ### Code
30
+
31
+ ```ruby
32
+ require "benchmark/sweet"
33
+ require "active_support/all"
34
+ require "more_core_extensions/all" # [].tabelize
35
+
36
+ NSTRING = nil
37
+ DELIMITER='/'.freeze
38
+ STRING="ab/cd/ef/gh".freeze
39
+
40
+ Benchmark.items(metrics: %w(ips memsize)) do |x|
41
+ x.metadata data: "nil" do
42
+ x.report("to_s.split") { NSTRING.to_s.split(DELIMITER) }
43
+ x.report("?split:[]") { NSTRING ? NSTRING.split(DELIMITER) : [] }
44
+ end
45
+ x.metadata data: "str" do
46
+ x.report("to_s.split") { STRING.to_s.split(DELIMITER) }
47
+ x.report("?split:[]") { STRING ? STRING.split(DELIMITER) : [] }
48
+ end
49
+
50
+ # partition the data by data value (nil vs string)
51
+ # that way we're not comparing a split on a nil vs a split on a populated string
52
+ compare_by :data
53
+
54
+ # each row is a different method (via `row: :method`)
55
+ # each column is by data type (via `column: :data` - specified via `metadata data: "nil"`)
56
+ x.report_with grouping: :metric, sort: true, row: :method, column: :data
57
+ end
58
+ ```
59
+
60
+ #### `compare_by`
61
+
62
+ The code takes a different amount of time to process a `nil` vs a string. So the values
63
+ are given metadata with the appropriate data used (i.e.: `metadata data: "nil"`).
64
+ The benchmark is then told that the results need to be partitioned by `data` (i.e.: `compare_by :data`).
65
+
66
+ The multipliers are only comparing values with the same data value and do not compare `"string"` values with `"nil"` values.
67
+
68
+ Values for method labels (e.g.: `"to_s.split"`) and metrics (e.g.: `ips`) are already part of the partition.
69
+
70
+ #### `grouping`
71
+
72
+ In this example, each metric is considered distinct so each metric is given a
73
+ unique table (i.e.: `grouping: :metric`)
74
+
75
+ Metrics of `ips` and `memsize` are calculated (i.e.: `metrics: %w(ips memsize)`)
76
+
77
+ #### `row`
78
+
79
+ A different method is given per row (i.e. `row: :method`)
80
+
81
+ #### `column`
82
+
83
+ The other axis for the columns is the type of data passed (i.e.: `column: :data`)
84
+ This is not a native value, it is specified when the items are specified (e.g.:`metadata data: "nil"`)
85
+
86
+ ### example output
87
+
88
+ #### metric ips
89
+
90
+ method | nil | str
91
+ ------------|-----------------------|---------------
92
+ `?split:[]` | 7090539.2 i/s | 1322010.9 i/s
93
+ `to_s.split`| 3703981.6 i/s - 1.91x | 1311153.9 i/s
94
+
95
+ #### metric memsize
96
+
97
+ method | nil | str
98
+ ------------|--------------------|-------------
99
+ `?split:[]` | 40.0 bytes | 360.0 bytes
100
+ `to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
101
+
102
+
103
+ ## Example 2
104
+
105
+ ### Code
106
+
107
+ ```ruby
108
+ require "benchmark/sweet"
109
+ require "active_support/all"
110
+ require "more_core_extensions/all" # [].tabelize
111
+
112
+ NSTRING = nil
113
+ DELIMITER='/'.freeze
114
+ STRING="ab/cd/ef/gh".freeze
115
+
116
+ Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 1, quiet: false, force: ENV["FORCE"] == "true") do |x|
117
+ x.metadata version: RUBY_VERSION
118
+ x.metadata data: "nil" do
119
+ x.report("to_s.split") { NSTRING.to_s.split(DELIMITER) }
120
+ x.report("?split:[]") { NSTRING ? NSTRING.split(DELIMITER) : [] }
121
+ end
122
+ x.metadata data: "str" do
123
+ x.report("to_s.split") { STRING.to_s.split(DELIMITER) }
124
+ x.report("?split:[]") { STRING ? STRING.split(DELIMITER) : [] }
125
+ end
126
+
127
+ # partition the data by ruby version and data present
128
+ # that way we're not comparing a split on a nil vs a split on a populated string
129
+ x.compare_by :version, :data
130
+ if ENV["CONDENSED"].to_s == "true"
131
+ x.report_with grouping: :version, sort: true, row: :method, column: [:data, :metric]
132
+ else
133
+ x.report_with grouping: [:version, :metric], sort: true, row: :method, column: :data
134
+ end
135
+
136
+ x.save_file $PROGRAM_NAME.sub(/\.rb$/, '.json')
137
+ end
138
+ ```
139
+
140
+ #### `save_file`
141
+
142
+ Creates a json save file which saves the timings across multiple runs.
143
+ This is used along with the `version` metadata to record different results per ruby version.
144
+ Another common use is to record ActiveRecord version or a gem's version.
145
+
146
+ This is run with two different versions of ruby.
147
+ Interim values are stored in the save_file.
148
+
149
+ Running with environment variable `FORCE` will force running this again. (i.e.: `force: ENV["force"] == true`)
150
+
151
+ Depending upon the environment variable `CONDENSED`, there are two types of output.
152
+
153
+ #### `compare_by`
154
+
155
+ We introduce `version` as metadata for the tests. Adding `version` to the comparison
156
+ says that we should only compare values for the same version of ruby (along with the same data).
157
+
158
+ If you note `version` and `data` are not reserved words, instead, they are just what metadata we
159
+ decided to pass in.
160
+
161
+ ### Example 2 output
162
+
163
+ #### [:version, :metric] 2.3.7_ips
164
+
165
+ method | nil | str
166
+ ------------|-----------------------|---------------
167
+ `?split:[]`| 10146134.2 i/s | 1284159.2 i/s
168
+ `to_s.split`| 4232772.3 i/s - 2.40x | 1258665.8 i/s
169
+
170
+ #### [:version, :metric] 2.3.7_memsize
171
+
172
+ method | nil | str
173
+ ------------|--------------------|-------------
174
+ `?split:[]`| 40.0 bytes | 360.0 bytes
175
+ `to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
176
+
177
+ #### [:version, :metric] 2.4.6_ips
178
+
179
+ method | nil | str
180
+ ------------|-----------------------|---------------
181
+ `?split:[]`| 10012873.4 i/s | 1377320.5 i/s
182
+ `to_s.split`| 4557456.3 i/s - 2.20x | 1350562.6 i/s
183
+
184
+ #### [:version, :metric] 2.4.6_memsize
185
+
186
+ method | nil | str
187
+ ------------|--------------------|-------------
188
+ `?split:[]`| 40.0 bytes | 360.0 bytes
189
+ `to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
190
+
191
+ #### [:version, :metric] 2.5.5_ips
192
+
193
+ method | nil | str
194
+ ------------|-----------------------|---------------
195
+ `?split:[]`| 7168109.1 i/s | 1357046.0 i/s
196
+ `to_s.split`| 3779969.3 i/s - 1.90x | 1328072.4 i/s
197
+
198
+ #### [:version, :metric] 2.5.5_memsize
199
+
200
+ method | nil | str
201
+ ------------|--------------------|-------------
202
+ `?split:[]`| 40.0 bytes | 360.0 bytes
203
+ `to_s.split`| 80.0 bytes - 2.00x | 360.0 bytes
204
+
205
+
206
+ ### Example 2 CONDENSED output
207
+
208
+ running with `CONDENSED=true` calls with a different `report_with`
209
+
210
+ As you might notice, the number of objects created for the runs are the same.
211
+ But for some reason, the `nil` split case is slower for ruby 2.5.5.
212
+
213
+ #### version 2.3.7
214
+
215
+ method | nil_ips | str_ips | nil_memsize | str_memsize
216
+ ------------|-----------------------|---------------|--------------------|-------------
217
+ `?split:[]`| 10146134.2 i/s | 1284159.2 i/s | 40.0 bytes | 360.0 bytes
218
+ `to_s.split`| 4232772.3 i/s - 2.40x | 1258665.8 i/s | 80.0 bytes - 2.00x | 360.0 bytes
219
+
220
+ #### version 2.4.6
221
+
222
+ method | nil_ips | str_ips | nil_memsize | str_memsize
223
+ ------------|-----------------------|---------------|--------------------|-------------
224
+ `?split:[]`| 10012873.4 i/s | 1377320.5 i/s | 40.0 bytes | 360.0 bytes
225
+ `to_s.split`| 4557456.3 i/s - 2.20x | 1350562.6 i/s | 80.0 bytes - 2.00x | 360.0 bytes
226
+
227
+ #### version 2.5.5
228
+
229
+ method | nil_ips | str_ips | nil_memsize | str_memsize
230
+ ------------|-----------------------|---------------|--------------------|-------------
231
+ `?split:[]`| 7168109.1 i/s | 1357046.0 i/s | 40.0 bytes | 360.0 bytes
232
+ `to_s.split`| 3779969.3 i/s - 1.90x | 1328072.4 i/s | 80.0 bytes - 2.00x | 360.0 bytes
233
+
234
+ ## Development
235
+
236
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
237
+
238
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
239
+
240
+ ## Contributing
241
+
242
+ Bug reports and pull requests are welcome on GitHub at https://github.com/kbrock/benchmark-sweet.
243
+
244
+ ## License
245
+
246
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,48 @@
1
+
2
+ lib = File.expand_path("../lib", __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require "benchmark/sweet/version"
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "benchmark-sweet"
8
+ spec.version = Benchmark::Sweet::VERSION
9
+ spec.authors = ["Keenan Brock"]
10
+ spec.email = ["keenan@thebrocks.net"]
11
+
12
+ spec.summary = %q{Suite to run multiple benchmarks}
13
+ spec.description = <<-EOF
14
+ Benchmark Sweet is a suite to run multiple kinds of metrics.
15
+ It can be configured to run memory, sql query, and ips benchmarks on a common set of code.
16
+ This data can be collected across multiple runs of the code, to support multiple ruby or
17
+ gem versions.
18
+ This also generates more complex comparisons
19
+ EOF
20
+
21
+ spec.homepage = "https://github.com/kbrock/benchmark-sweet"
22
+ spec.license = "MIT"
23
+
24
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
25
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
26
+ if spec.respond_to?(:metadata)
27
+ spec.metadata["homepage_uri"] = spec.homepage
28
+ spec.metadata["source_code_uri"] = "http://github.com/kbrock/benchmark-sweet"
29
+ spec.metadata["changelog_uri"] = "http://github.com/kbrock/benchmark-sweet/CHANGELOG.md"
30
+ end
31
+
32
+ # Specify which files should be added to the gem when it is released.
33
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
34
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
35
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
36
+ end
37
+ spec.require_paths = ["lib"]
38
+
39
+ spec.add_development_dependency "bundler", "~> 2.1.4"
40
+ spec.add_development_dependency "rake", ">= 12.3.3"
41
+ spec.add_development_dependency "rspec", "~> 3.0"
42
+ spec.add_development_dependency "activerecord"
43
+
44
+ spec.add_runtime_dependency "benchmark-ips", "~> 2.8.2"
45
+ spec.add_runtime_dependency "memory_profiler", "~> 0.9.0"
46
+ spec.add_runtime_dependency "more_core_extensions" # for [].tabelize - want to remove
47
+ spec.add_runtime_dependency "activesupport" # for {}.symbolize_keys! - want to remove
48
+ end
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "benchmark/sweet"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,84 @@
1
+ require "benchmark/sweet"
2
+ require "active_support/all"
3
+ require "more_core_extensions/all" # [].tabelize
4
+
5
+ # output:
6
+ #
7
+ # [:version] 2.3.7
8
+ #
9
+ # method | nil_ips | str_ips | nil_memsize | str_memsize
10
+ # ------------+-----------------------+---------------+--------------------+-------------
11
+ # ?split:[] | 51574246.6 i/s | 1434943.3 i/s | 40.0 bytes | 360.0 bytes
12
+ # to_s.split | 5707766.1 i/s - 9.04x | 1411708.0 i/s | 80.0 bytes - 2.00x | 360.0 bytes
13
+ #
14
+ # [:version] 2.4.4
15
+ #
16
+ # method | nil_ips | str_ips | nil_memsize | str_memsize
17
+ # ------------+-----------------------+---------------+--------------------+-------------
18
+ # ?split:[] | 51740193.6 i/s | 1411882.2 i/s | 40.0 bytes | 360.0 bytes
19
+ # to_s.split | 5887968.5 i/s - 8.79x | 1347822.1 i/s | 80.0 bytes - 2.00x | 360.0 bytes
20
+ #
21
+ #
22
+ #
23
+ # (CONDENSED=true) output:
24
+ #
25
+ # [:metric, :version] ips_2.3.7
26
+ #
27
+ # method | nil | str
28
+ # -------------+------------------------+---------------
29
+ # ?split:[] | 51825798.8 i/s | 1407946.4 i/s
30
+ # &&split||[] | 46730725.8 i/s - 1.11x | 1413355.3 i/s
31
+ # to_s.split | 5685237.1 i/s - 9.12x | 1396494.3 i/s
32
+ #
33
+ # [:metric, :version] ips_2.4.4
34
+ #
35
+ # method | nil | str
36
+ # -------------+------------------------+---------------
37
+ # ?split:[] | 51559454.4 i/s | 1438780.8 i/s
38
+ # &.split||[] | 46446196.0 i/s - 1.11x | 1437665.3 i/s
39
+ # &&split||[] | 43356335.6 i/s - 1.19x | 1434466.6 i/s
40
+ # to_s.split | 5835694.7 i/s - 8.84x | 1427819.0 i/s
41
+ #
42
+ # [:metric, :version] memsize_2.3.7
43
+ #
44
+ # method | nil | str
45
+ # -------------+--------------------+-------------
46
+ # ?split:[] | 40.0 bytes | 360.0 bytes
47
+ # &&split||[] | 40.0 bytes | 360.0 bytes
48
+ # to_s.split | 80.0 bytes - 2.00x | 360.0 bytes
49
+ #
50
+ # [:metric, :version] memsize_2.4.4
51
+ #
52
+ # method | nil | str
53
+ # -------------+--------------------+-------------
54
+ # ?split:[] | 40.0 bytes | 360.0 bytes
55
+ # &&split||[] | 40.0 bytes | 360.0 bytes
56
+ # &.split||[] | 40.0 bytes | 360.0 bytes
57
+ # to_s.split | 80.0 bytes - 2.00x | 360.0 bytes
58
+
59
+ NSTRING = nil
60
+ DELIMITER='/'.freeze
61
+ STRING="ab/cd/ef/gh".freeze
62
+
63
+ Benchmark.items(metrics: %w(ips memsize), memory: 3, warmup: 1, time: 1, quiet: false, force: ENV["FORCE"] == "true") do |x|
64
+ x.metadata version: RUBY_VERSION
65
+ x.metadata data: "nil" do
66
+ x.report("to_s.split", "NSTRING.to_s.split(DELIMITER)")
67
+ x.report("?split:[]", "NSTRING ? NSTRING.split(DELIMITER) : []")
68
+ end
69
+ x.metadata data: :str do
70
+ x.report("to_s.split", "STRING.to_s.split(DELIMITER)")
71
+ x.report("?split:[]", "STRING ? STRING.split(DELIMITER) : []")
72
+ end
73
+
74
+ # partition the data by ruby version and data present
75
+ # that way we're not comparing a split on a nil vs a split on a populated string
76
+ x.compare_by :version, :data
77
+ if ENV["CONDENSED"].to_s == "true"
78
+ x.report_with grouping: :version, sort: true, row: :method, column: [:data, :metric]
79
+ else
80
+ x.report_with grouping: [:version, :metric], sort: true, row: :method, column: :data
81
+ end
82
+
83
+ x.save_file (ENV["SAVE_FILE"] == "true") ? $0.sub(/\.rb$/, '.json') : ENV["SAVE_FILE"] if ENV["SAVE_FILE"]
84
+ end