benchmark-ips 2.6.1 → 2.7.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2edc0877b8b37420698e685b9db778b2bad7c094
4
- data.tar.gz: d034c9416613ed22534e5a2e7b4da8c989313fb8
3
+ metadata.gz: 122a521904ed1263289b5f68e76c72515830447e
4
+ data.tar.gz: 6b60d87ce82746370a1d0bff1f49be8175b2a5be
5
5
  SHA512:
6
- metadata.gz: f807342d963e71a638e77db1f1eeb15f1122fffc996a5e98874d7691821407000d83eba71b081fb0951ef0a6cd10f04e1c951698d0486d6b61561150c6b58631
7
- data.tar.gz: a47ebd2708d984bdbf960ca50937021f5d2439a216cebf9ec23fe68178b4a6acfdae1e45588748ff182211311950f775c0ab5a4620ec093707c02a8bbf685e7c
6
+ metadata.gz: 3e3588122227405ccb4ce7376e431ee630ca87779633cecdf24b82ba672610cbafc92ff82685b05accda763a54a0c54b50fe2a42763c5f45a9a2171984aaec69
7
+ data.tar.gz: cf47674032d38ac758b371edae0b1ffeceaac07ad9f7fc10d8413ffeb2db173395cf90d187b41293d122c50f6c2a9c68776b1e15c6c0c425549928d697d11485
@@ -1,3 +1,22 @@
1
+ === 2.7.0 / 2016-08-05
2
+
3
+ * 1 minor features:
4
+ * Add support for confidence intervals
5
+
6
+ * 1 bug fixes:
7
+ * Cleanup a few coding patterns
8
+
9
+ * 2 doc fixes:
10
+ * Add infos about benchark.fyi to Readme
11
+ * Remove ancient releases
12
+
13
+ * 3 merged PRs:
14
+ * Merge pull request #65 from kbrock/fixup_inject
15
+ * Merge pull request #67 from benoittgt/master
16
+ * Merge pull request #69 from chrisseaton/kalibera-confidence-intervals
17
+
18
+ === MISSING 2.6.0 and 2.6.1
19
+
1
20
  === 2.5.0 / 2016-02-14
2
21
 
3
22
  * 1 minor feature:
data/README.md CHANGED
@@ -144,10 +144,10 @@ are independent of each other. You can do this with the `hold!` command.
144
144
 
145
145
  ```ruby
146
146
  Benchmark.ips do |x|
147
-
147
+
148
148
  # Hold results between multiple invocations of Ruby
149
149
  x.hold! 'filename'
150
-
150
+
151
151
  end
152
152
  ```
153
153
 
@@ -172,9 +172,58 @@ Benchmark.ips do |x|
172
172
  x.config(:iterations => 3)
173
173
 
174
174
  # or
175
-
175
+
176
176
  x.iterations = 3
177
+
178
+ end
179
+ ```
180
+
181
+ ### Online sharing
182
+
183
+ If you want to share quickly your benchmark result with others. Run you benchmark
184
+ with `SHARE=1` argument. I.e.: `SHARE=1 ruby my_benchmark.rb`.
185
+ Result will be sent to [benchmark.fyi](https://benchmark.fyi/) and benchmark-ips
186
+ will display the link to share the benchmark's result.
187
+
188
+ ### Advanced Statistics
189
+
190
+ By default, the margin of error shown is plus-minus one standard deviation. If
191
+ a more advanced statistical test is wanted, a bootstrap confidence interval
192
+ can be calculated instead. A bootstrap confidence interval has the advantages of
193
+ arguably being more mathematically sound for this application than a standard
194
+ deviation, it additionally produces an error for relative slowdowns, which the
195
+ standard deviation does not, and it is arguably more intuitive and actionable.
196
+
197
+ When a bootstrap confidence interval is used, a median of the interval is used
198
+ rather than the mean of the samples, which is what you get with the default
199
+ standard deviation.
200
+
201
+ The bootstrap confidence interval used is the one described by Tomas Kalibera.
202
+ Note that for this technique to be valid your benchmark should have reached a
203
+ non-periodic steady state with statistically independent samples (it should
204
+ have warmed up) by the time measurements start.
205
+
206
+ Using a bootstrap confidence internal requires that the 'kalibera' gem is
207
+ installed separately. This gem is not a formal dependency, as by default it is
208
+ not needed.
209
+
210
+ ```
211
+ gem install kalibera
212
+ ```
213
+
214
+ ```ruby
215
+ Benchmark.ips do |x|
216
+
217
+ # The default is :stats => :sd, which doesn't have a configurable confidence
218
+ x.config(:stats => :bootstrap, :confidence => 95)
219
+
220
+ # or
221
+
222
+ x.stats = :bootstrap
223
+ x.confidence = 95
177
224
 
225
+ # confidence is 95% by default, so it can be omitted
226
+
178
227
  end
179
228
  ```
180
229
 
@@ -1,3 +1,5 @@
1
+ # encoding: utf-8
2
+
1
3
  module Benchmark
2
4
  # Functionality of performaing comparison between reports.
3
5
  #
@@ -32,33 +34,40 @@ module Benchmark
32
34
  def compare(*entries)
33
35
  return if entries.size < 2
34
36
 
35
- sorted = entries.sort_by(&:ips).reverse
37
+ sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
36
38
 
37
39
  best = sorted.shift
38
40
 
39
41
  $stdout.puts "\nComparison:"
40
42
 
41
- $stdout.printf "%20s: %10.1f i/s\n", best.label, best.ips
43
+ $stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
42
44
 
43
45
  sorted.each do |report|
44
46
  name = report.label.to_s
45
47
 
46
- $stdout.printf "%20s: %10.1f i/s - ", name, report.ips
48
+ $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
47
49
 
48
- best_low = best.ips - best.ips_sd
49
- report_high = report.ips + report.ips_sd
50
+ best_low = best.stats.central_tendency - best.stats.error
51
+ report_high = report.stats.central_tendency + report.stats.error
50
52
  overlaps = report_high > best_low
51
53
 
52
54
  if overlaps
53
55
  $stdout.print "same-ish: difference falls within error"
54
56
  else
55
- x = (best.ips.to_f / report.ips.to_f)
56
- $stdout.printf "%.2fx slower", x
57
+ slowdown, error = report.stats.slowdown(best.stats)
58
+ $stdout.printf "%.2fx ", slowdown
59
+ if error
60
+ $stdout.printf " (± %.2f)", error
61
+ end
62
+ $stdout.print " slower"
57
63
  end
58
64
 
59
65
  $stdout.puts
60
66
  end
61
67
 
68
+ footer = best.stats.footer
69
+ $stdout.puts footer.rjust(40) if footer
70
+
62
71
  $stdout.puts
63
72
  end
64
73
  end
@@ -1,6 +1,8 @@
1
1
  # encoding: utf-8
2
2
  require 'benchmark/timing'
3
3
  require 'benchmark/compare'
4
+ require 'benchmark/ips/stats/sd'
5
+ require 'benchmark/ips/stats/bootstrap'
4
6
  require 'benchmark/ips/report'
5
7
  require 'benchmark/ips/job/entry'
6
8
  require 'benchmark/ips/job/stdout_report'
@@ -13,10 +15,10 @@ module Benchmark
13
15
  module IPS
14
16
 
15
17
  # Benchmark-ips Gem version.
16
- VERSION = "2.6.1"
18
+ VERSION = "2.7.0"
17
19
 
18
20
  # CODENAME of current version.
19
- CODENAME = "Sharing is Caring"
21
+ CODENAME = "Cultivating Confidence"
20
22
 
21
23
  # Measure code in block, each code's benchmarked result will display in
22
24
  # iteration per second with standard deviation in given time.
@@ -42,6 +42,14 @@ module Benchmark
42
42
  # @return [Integer]
43
43
  attr_accessor :iterations
44
44
 
45
+ # Statistics model.
46
+ # @return [Object]
47
+ attr_accessor :stats
48
+
49
+ # Confidence.
50
+ # @return [Integer]
51
+ attr_accessor :confidence
52
+
45
53
  # Instantiate the Benchmark::IPS::Job.
46
54
  # @option opts [Benchmark::Suite] (nil) :suite Specify Benchmark::Suite.
47
55
  # @option opts [Boolean] (false) :quiet Suppress the printing of information.
@@ -61,6 +69,10 @@ module Benchmark
61
69
  @warmup = 2
62
70
  @time = 5
63
71
  @iterations = 1
72
+
73
+ # Default statistical model
74
+ @stats = :sd
75
+ @confidence = 95
64
76
  end
65
77
 
66
78
  # Job configuration options, set +@warmup+ and +@time+.
@@ -72,6 +84,8 @@ module Benchmark
72
84
  @time = opts[:time] if opts[:time]
73
85
  @suite = opts[:suite] if opts[:suite]
74
86
  @iterations = opts[:iterations] if opts[:iterations]
87
+ @stats = opts[:stats] if opts[:stats]
88
+ @confidence = opts[:confidence] if opts[:confidence]
75
89
  end
76
90
 
77
91
  # Return true if job needs to be compared.
@@ -133,8 +147,7 @@ module Benchmark
133
147
  # @return [Integer] Cycles per 100ms.
134
148
  def cycles_per_100ms time_msec, iters
135
149
  cycles = ((MICROSECONDS_PER_100MS / time_msec) * iters).to_i
136
- cycles = 1 if cycles <= 0
137
- cycles
150
+ cycles <= 0 ? 1 : cycles
138
151
  end
139
152
 
140
153
  # Calculate the time difference of before and after in microseconds.
@@ -179,6 +192,8 @@ module Benchmark
179
192
  @iterations.times do |n|
180
193
  held = run_benchmark
181
194
  end
195
+
196
+ @stdout.footer if @stdout
182
197
 
183
198
  if held
184
199
  puts
@@ -225,7 +240,7 @@ module Benchmark
225
240
  if hold? && @held_results && @held_results.key?(item.label)
226
241
  result = @held_results[item.label]
227
242
  create_report(item.label, result['measured_us'], result['iter'],
228
- result['avg_ips'], result['sd_ips'], result['cycles'])
243
+ create_stats(result['samples']), result['cycles'])
229
244
  next
230
245
  end
231
246
 
@@ -259,16 +274,13 @@ module Benchmark
259
274
 
260
275
  final_time = before
261
276
 
262
- measured_us = measurements_us.inject(0) { |a,i| a + i }
277
+ measured_us = measurements_us.inject(:+)
263
278
 
264
- all_ips = measurements_us.map { |time_us|
279
+ samples = measurements_us.map { |time_us|
265
280
  iterations_per_sec cycles, time_us
266
281
  }
267
282
 
268
- avg_ips = Timing.mean(all_ips)
269
- sd_ips = Timing.stddev(all_ips, avg_ips).round
270
-
271
- rep = create_report(item.label, measured_us, iter, avg_ips, sd_ips, cycles)
283
+ rep = create_report(item.label, measured_us, iter, create_stats(samples), cycles)
272
284
 
273
285
  if (final_time - target).abs >= (@time.to_f * MAX_TIME_SKEW)
274
286
  rep.show_total_time!
@@ -284,8 +296,7 @@ module Benchmark
284
296
  :item => item.label,
285
297
  :measured_us => measured_us,
286
298
  :iter => iter,
287
- :avg_ips => avg_ips,
288
- :sd_ips => sd_ips,
299
+ :samples => samples,
289
300
  :cycles => cycles
290
301
  })
291
302
  f.write "\n"
@@ -302,6 +313,17 @@ module Benchmark
302
313
  false
303
314
  end
304
315
 
316
+ def create_stats(samples)
317
+ case @stats
318
+ when :sd
319
+ Stats::SD.new(samples)
320
+ when :bootstrap
321
+ Stats::Bootstrap.new(samples, @confidence)
322
+ else
323
+ raise "unknown stats #{@stats}"
324
+ end
325
+ end
326
+
305
327
  # Run comparison of entries in +@full_report+.
306
328
  def run_comparison
307
329
  @full_report.run_comparison if compare?
@@ -316,12 +338,11 @@ module Benchmark
316
338
  # @param label [String] Report item label.
317
339
  # @param measured_us [Integer] Measured time in microsecond.
318
340
  # @param iter [Integer] Iterations.
319
- # @param avg_ips [Float] Average iterations per second.
320
- # @param sd_ips [Float] Standard deviation iterations per second.
341
+ # @param samples [Array<Float>] Sampled iterations per second.
321
342
  # @param cycles [Integer] Number of Cycles.
322
343
  # @return [Report::Entry] Entry with data.
323
- def create_report(label, measured_us, iter, avg_ips, sd_ips, cycles)
324
- @full_report.add_entry label, measured_us, iter, avg_ips, sd_ips, cycles
344
+ def create_report(label, measured_us, iter, samples, cycles)
345
+ @full_report.add_entry label, measured_us, iter, samples, cycles
325
346
  end
326
347
  end
327
348
  end
@@ -27,6 +27,12 @@ module Benchmark
27
27
 
28
28
  def add_report(item, caller)
29
29
  $stdout.puts " #{item.body}"
30
+ @last_item = item
31
+ end
32
+
33
+ def footer
34
+ footer = @last_item.stats.footer
35
+ $stdout.puts footer.rjust(40) if footer
30
36
  end
31
37
 
32
38
  private
@@ -13,15 +13,13 @@ module Benchmark
13
13
  # @param [#to_s] label Label of entry.
14
14
  # @param [Integer] us Measured time in microsecond.
15
15
  # @param [Integer] iters Iterations.
16
- # @param [Float] ips Iterations per second.
17
- # @param [Float] ips_sd Standard deviation of iterations per second.
16
+ # @param [Object] stats Statistics.
18
17
  # @param [Integer] cycles Number of Cycles.
19
- def initialize(label, us, iters, ips, ips_sd, cycles)
18
+ def initialize(label, us, iters, stats, cycles)
20
19
  @label = label
21
20
  @microseconds = us
22
21
  @iterations = iters
23
- @ips = ips
24
- @ips_sd = ips_sd
22
+ @stats = stats
25
23
  @measurement_cycle = cycles
26
24
  @show_total_time = false
27
25
  end
@@ -38,13 +36,9 @@ module Benchmark
38
36
  # @return [Integer] number of iterations.
39
37
  attr_reader :iterations
40
38
 
41
- # Iterations per second.
42
- # @return [Float] number of iterations per second.
43
- attr_reader :ips
44
-
45
- # Standard deviation of iteration per second.
46
- # @return [Float] standard deviation of iteration per second.
47
- attr_reader :ips_sd
39
+ # Statistical summary of samples.
40
+ # @return [Object] statisical summary.
41
+ attr_reader :stats
48
42
 
49
43
  # Number of Cycles.
50
44
  # @return [Integer] number of cycles.
@@ -65,8 +59,8 @@ module Benchmark
65
59
 
66
60
  # Return entry's standard deviation of iteration per second in percentage.
67
61
  # @return [Float] +@ips_sd+ in percentage.
68
- def stddev_percentage
69
- 100.0 * (@ips_sd.to_f / @ips.to_f)
62
+ def error_percentage
63
+ 100.0 * (@stats.error.to_f / @stats.central_tendency)
70
64
  end
71
65
 
72
66
  alias_method :runtime, :seconds
@@ -78,7 +72,7 @@ module Benchmark
78
72
  def body
79
73
  case Benchmark::IPS.options[:format]
80
74
  when :human
81
- left = "%s (±%4.1f%%) i/s" % [Helpers.scale(ips), stddev_percentage]
75
+ left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
82
76
  iters = Helpers.scale(@iterations)
83
77
 
84
78
  if @show_total_time
@@ -87,7 +81,7 @@ module Benchmark
87
81
  left.ljust(20) + (" - %s" % iters)
88
82
  end
89
83
  else
90
- left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
84
+ left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
91
85
 
92
86
  if @show_total_time
93
87
  left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
@@ -131,12 +125,11 @@ module Benchmark
131
125
  # @param label [String] Entry label.
132
126
  # @param microseconds [Integer] Measured time in microsecond.
133
127
  # @param iters [Integer] Iterations.
134
- # @param ips [Float] Average Iterations per second.
135
- # @param ips_sd [Float] Standard deviation of iterations per second.
128
+ # @param stats [Object] Statistical results.
136
129
  # @param measurement_cycle [Integer] Number of cycles.
137
130
  # @return [Report::Entry] Last added entry.
138
- def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
139
- entry = Entry.new(label, microseconds, iters, ips, ips_sd, measurement_cycle)
131
+ def add_entry label, microseconds, iters, stats, measurement_cycle
132
+ entry = Entry.new(label, microseconds, iters, stats, measurement_cycle)
140
133
  @entries.delete_if { |e| e.label == label }
141
134
  @entries << entry
142
135
  entry
@@ -155,8 +148,10 @@ module Benchmark
155
148
  @data ||= @entries.collect do |entry|
156
149
  {
157
150
  :name => entry.label,
158
- :ips => entry.ips,
159
- :stddev => entry.ips_sd,
151
+ :central_tendency => entry.stats.central_tendency,
152
+ :ips => entry.stats.central_tendency, # for backwards compatibility
153
+ :error => entry.stats.error,
154
+ :stddev => entry.stats.error, # for backwards compatibility
160
155
  :microseconds => entry.microseconds,
161
156
  :iterations => entry.iterations,
162
157
  :cycles => entry.measurement_cycle,
@@ -8,7 +8,7 @@ module Benchmark
8
8
  # @param [Array] samples Samples to calculate mean.
9
9
  # @return [Float] Mean of given samples.
10
10
  def self.mean(samples)
11
- sum = samples.inject(0) { |acc, i| acc + i }
11
+ sum = samples.inject(:+)
12
12
  sum / samples.size
13
13
  end
14
14
 
@@ -31,20 +31,6 @@ module Benchmark
31
31
  Math.sqrt variance(samples, m)
32
32
  end
33
33
 
34
- # Resample mean of given samples.
35
- # @param [Integer] resample_times Resample times, defaults to 100.
36
- # @return [Array] Resampled samples.
37
- def self.resample_mean(samples, resample_times=100)
38
- resamples = []
39
-
40
- resample_times.times do
41
- resample = samples.map { samples[rand(samples.size)] }
42
- resamples << Timing.mean(resample)
43
- end
44
-
45
- resamples
46
- end
47
-
48
34
  # Recycle used objects by starting Garbage Collector.
49
35
  def self.clean_env
50
36
  # rbx
@@ -66,7 +52,7 @@ module Benchmark
66
52
 
67
53
  # Add one second to the time represenetation
68
54
  def self.add_second(t, s)
69
- return t + (s * MICROSECONDS_PER_SECOND)
55
+ t + (s * MICROSECONDS_PER_SECOND)
70
56
  end
71
57
 
72
58
  # Return the number of microseconds between the 2 moments
@@ -81,7 +67,7 @@ module Benchmark
81
67
 
82
68
  # Add one second to the time represenetation
83
69
  def self.add_second(t, s)
84
- return t + s
70
+ t + s
85
71
  end
86
72
 
87
73
  # Return the number of microseconds between the 2 moments
@@ -55,11 +55,11 @@ class TestBenchmarkIPS < Minitest::Test
55
55
 
56
56
  assert_equal "sleep 0.25", rep1.label
57
57
  assert_equal 4, rep1.iterations
58
- assert_in_delta 4.0, rep1.ips, 0.2
58
+ assert_in_delta 4.0, rep1.stats.central_tendency, 0.2
59
59
 
60
60
  assert_equal "sleep 0.05", rep2.label
61
61
  assert_in_delta 20.0, rep2.iterations.to_f, 1.0
62
- assert_in_delta 20.0, rep2.ips, 2.0
62
+ assert_in_delta 20.0, rep2.stats.central_tendency, 2.0
63
63
  end
64
64
 
65
65
  def test_ips_alternate_config
@@ -73,7 +73,7 @@ class TestBenchmarkIPS < Minitest::Test
73
73
 
74
74
  assert_equal "sleep 0.25", rep.label
75
75
  assert_equal 4, rep.iterations
76
- assert_in_delta 4.0, rep.ips, 0.4
76
+ assert_in_delta 4.0, rep.stats.central_tendency, 0.4
77
77
  end
78
78
 
79
79
  def test_ips_old_config
@@ -85,7 +85,7 @@ class TestBenchmarkIPS < Minitest::Test
85
85
 
86
86
  assert_equal "sleep 0.25", rep.label
87
87
  assert_equal 4, rep.iterations
88
- assert_in_delta 4.0, rep.ips, 0.2
88
+ assert_in_delta 4.0, rep.stats.central_tendency, 0.2
89
89
  end
90
90
 
91
91
  def test_ips_config_suite
@@ -112,7 +112,7 @@ class TestBenchmarkIPS < Minitest::Test
112
112
 
113
113
  assert_equal "sleep 0.25", rep.label
114
114
  assert_equal 4*5, rep.iterations
115
- assert_in_delta 4.0, rep.ips, 0.2
115
+ assert_in_delta 4.0, rep.stats.central_tendency, 0.2
116
116
  end
117
117
 
118
118
  def test_ips_report_using_symbol
@@ -124,7 +124,7 @@ class TestBenchmarkIPS < Minitest::Test
124
124
 
125
125
  assert_equal :sleep_a_quarter_second, rep.label
126
126
  assert_equal 4*5, rep.iterations
127
- assert_in_delta 4.0, rep.ips, 0.2
127
+ assert_in_delta 4.0, rep.stats.central_tendency, 0.2
128
128
  end
129
129
 
130
130
  def test_ips_default_data
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: benchmark-ips
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.6.1
4
+ version: 2.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Evan Phoenix
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-13 00:00:00.000000000 Z
11
+ date: 2016-08-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: minitest
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '5.8'
19
+ version: '5.9'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '5.8'
26
+ version: '5.9'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rdoc
29
29
  requirement: !ruby/object:Gem::Requirement