benchmark-ips 2.7.2 → 2.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Manifest.txt +0 -1
- data/README.md +10 -4
- data/lib/benchmark/compare.rb +5 -9
- data/lib/benchmark/ips.rb +74 -3
- data/lib/benchmark/ips/job.rb +94 -71
- data/lib/benchmark/ips/job/stdout_report.rb +6 -1
- data/lib/benchmark/ips/report.rb +7 -3
- data/lib/benchmark/ips/stats/bootstrap.rb +9 -6
- data/lib/benchmark/ips/stats/sd.rb +19 -11
- data/test/test_benchmark_ips.rb +24 -0
- metadata +17 -12
- data/Gemfile.lock +0 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 54899cb6088746f3b585d3e471795fa63c5fa06b9a452e7923d5211ea096eff1
|
4
|
+
data.tar.gz: 21f3a97a2971d77dbf33fa05a497f7c61499497da66b8aff26987c64d6e2de3c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b7dd9271181dc58b32864d0d60ecf58c8daf92dc024ab764bc07289a39650db30354d72fc8f586207c6f5ccfaccf72c41eb9ac75d4e7be7af7a18c60f6e7860
|
7
|
+
data.tar.gz: 11731f19a4fbc1abcc6e31ad6c5d4ea752f26d3a5ce9573b4e07011b568c455abfbd5a53b5e41408d7e931a5327d1c6c0a2200c25d10897b9d4111d45326c7d5
|
data/Manifest.txt
CHANGED
data/README.md
CHANGED
@@ -1,13 +1,14 @@
|
|
1
|
+
# benchmark-ips
|
2
|
+
|
3
|
+
* rdoc :: http://rubydoc.info/gems/benchmark-ips
|
4
|
+
* home :: https://github.com/evanphx/benchmark-ips
|
5
|
+
|
1
6
|
[![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
|
2
7
|
[![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
|
3
8
|
[![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
|
4
9
|
|
5
|
-
# benchmark-ips
|
6
|
-
|
7
10
|
* https://github.com/evanphx/benchmark-ips
|
8
11
|
|
9
|
-
* [documentation](http://rubydoc.info/gems/benchmark-ips)
|
10
|
-
|
11
12
|
## DESCRIPTION:
|
12
13
|
|
13
14
|
An iterations per second enhancement to Benchmark.
|
@@ -155,6 +156,11 @@ This will run only one benchmarks each time you run the command, storing
|
|
155
156
|
results in the specified file. The file is deleted when all results have been
|
156
157
|
gathered and the report is shown.
|
157
158
|
|
159
|
+
Alternatively, if you prefer a different approach, the `save!` command is
|
160
|
+
available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
|
161
|
+
the `examples/` directory.
|
162
|
+
|
163
|
+
|
158
164
|
### Multiple iterations
|
159
165
|
|
160
166
|
In some cases you may want to run multiple iterations of the warmup and
|
data/lib/benchmark/compare.rb
CHANGED
@@ -40,18 +40,14 @@ module Benchmark
|
|
40
40
|
|
41
41
|
$stdout.puts "\nComparison:"
|
42
42
|
|
43
|
-
$stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
|
43
|
+
$stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
|
44
44
|
|
45
45
|
sorted.each do |report|
|
46
46
|
name = report.label.to_s
|
47
|
-
|
47
|
+
|
48
48
|
$stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
|
49
|
-
|
50
|
-
|
51
|
-
report_high = report.stats.central_tendency + report.stats.error
|
52
|
-
overlaps = report_high > best_low
|
53
|
-
|
54
|
-
if overlaps
|
49
|
+
|
50
|
+
if report.stats.overlaps?(best.stats)
|
55
51
|
$stdout.print "same-ish: difference falls within error"
|
56
52
|
else
|
57
53
|
slowdown, error = report.stats.slowdown(best.stats)
|
@@ -61,7 +57,7 @@ module Benchmark
|
|
61
57
|
end
|
62
58
|
$stdout.print " slower"
|
63
59
|
end
|
64
|
-
|
60
|
+
|
65
61
|
$stdout.puts
|
66
62
|
end
|
67
63
|
|
data/lib/benchmark/ips.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'benchmark/timing'
|
3
3
|
require 'benchmark/compare'
|
4
|
+
require 'benchmark/ips/stats/stats_metric'
|
4
5
|
require 'benchmark/ips/stats/sd'
|
5
6
|
require 'benchmark/ips/stats/bootstrap'
|
6
7
|
require 'benchmark/ips/report'
|
@@ -15,10 +16,10 @@ module Benchmark
|
|
15
16
|
module IPS
|
16
17
|
|
17
18
|
# Benchmark-ips Gem version.
|
18
|
-
VERSION = "2.
|
19
|
+
VERSION = "2.8.0"
|
19
20
|
|
20
21
|
# CODENAME of current version.
|
21
|
-
CODENAME = "
|
22
|
+
CODENAME = "Tardy Turtle"
|
22
23
|
|
23
24
|
# Measure code in block, each code's benchmarked result will display in
|
24
25
|
# iteration per second with standard deviation in given time.
|
@@ -54,10 +55,17 @@ module Benchmark
|
|
54
55
|
|
55
56
|
yield job
|
56
57
|
|
57
|
-
job.load_held_results
|
58
|
+
job.load_held_results
|
58
59
|
|
59
60
|
job.run
|
60
61
|
|
62
|
+
if job.run_single? && job.all_results_have_been_run?
|
63
|
+
job.clear_held_results
|
64
|
+
else
|
65
|
+
job.save_held_results
|
66
|
+
puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
|
67
|
+
end
|
68
|
+
|
61
69
|
$stdout.sync = sync
|
62
70
|
job.run_comparison
|
63
71
|
job.generate_json
|
@@ -101,5 +109,68 @@ module Benchmark
|
|
101
109
|
end
|
102
110
|
end
|
103
111
|
|
112
|
+
##
|
113
|
+
# :singleton-method: ips
|
114
|
+
#
|
115
|
+
# require 'benchmark/ips'
|
116
|
+
#
|
117
|
+
# Benchmark.ips do |x|
|
118
|
+
# # Configure the number of seconds used during
|
119
|
+
# # the warmup phase (default 2) and calculation phase (default 5)
|
120
|
+
# x.config(:time => 5, :warmup => 2)
|
121
|
+
#
|
122
|
+
# # These parameters can also be configured this way
|
123
|
+
# x.time = 5
|
124
|
+
# x.warmup = 2
|
125
|
+
#
|
126
|
+
# # Typical mode, runs the block as many times as it can
|
127
|
+
# x.report("addition") { 1 + 2 }
|
128
|
+
#
|
129
|
+
# # To reduce overhead, the number of iterations is passed in
|
130
|
+
# # and the block must run the code the specific number of times.
|
131
|
+
# # Used for when the workload is very small and any overhead
|
132
|
+
# # introduces incorrectable errors.
|
133
|
+
# x.report("addition2") do |times|
|
134
|
+
# i = 0
|
135
|
+
# while i < times
|
136
|
+
# 1 + 2
|
137
|
+
# i += 1
|
138
|
+
# end
|
139
|
+
# end
|
140
|
+
#
|
141
|
+
# # To reduce overhead even more, grafts the code given into
|
142
|
+
# # the loop that performs the iterations internally to reduce
|
143
|
+
# # overhead. Typically not needed, use the |times| form instead.
|
144
|
+
# x.report("addition3", "1 + 2")
|
145
|
+
#
|
146
|
+
# # Really long labels should be formatted correctly
|
147
|
+
# x.report("addition-test-long-label") { 1 + 2 }
|
148
|
+
#
|
149
|
+
# # Compare the iterations per second of the various reports!
|
150
|
+
# x.compare!
|
151
|
+
# end
|
152
|
+
#
|
153
|
+
# This will generate the following report:
|
154
|
+
#
|
155
|
+
# Calculating -------------------------------------
|
156
|
+
# addition 71.254k i/100ms
|
157
|
+
# addition2 68.658k i/100ms
|
158
|
+
# addition3 83.079k i/100ms
|
159
|
+
# addition-test-long-label
|
160
|
+
# 70.129k i/100ms
|
161
|
+
# -------------------------------------------------
|
162
|
+
# addition 4.955M (± 8.7%) i/s - 24.155M
|
163
|
+
# addition2 24.011M (± 9.5%) i/s - 114.246M
|
164
|
+
# addition3 23.958M (±10.1%) i/s - 115.064M
|
165
|
+
# addition-test-long-label
|
166
|
+
# 5.014M (± 9.1%) i/s - 24.545M
|
167
|
+
#
|
168
|
+
# Comparison:
|
169
|
+
# addition2: 24011974.8 i/s
|
170
|
+
# addition3: 23958619.8 i/s - 1.00x slower
|
171
|
+
# addition-test-long-label: 5014756.0 i/s - 4.79x slower
|
172
|
+
# addition: 4955278.9 i/s - 4.85x slower
|
173
|
+
#
|
174
|
+
# See also Benchmark::IPS
|
104
175
|
extend Benchmark::IPS # make ips available as module-level method
|
105
176
|
end
|
data/lib/benchmark/ips/job.rb
CHANGED
@@ -58,11 +58,12 @@ module Benchmark
|
|
58
58
|
@stdout = opts[:quiet] ? nil : StdoutReport.new
|
59
59
|
@list = []
|
60
60
|
@compare = false
|
61
|
+
@run_single = false
|
61
62
|
@json_path = false
|
62
63
|
@held_path = nil
|
63
64
|
@held_results = nil
|
64
65
|
|
65
|
-
@timing =
|
66
|
+
@timing = Hash.new 1 # default to 1 in case warmup isn't run
|
66
67
|
@full_report = Report.new
|
67
68
|
|
68
69
|
# Default warmup and calculation time in seconds.
|
@@ -94,7 +95,7 @@ module Benchmark
|
|
94
95
|
@compare
|
95
96
|
end
|
96
97
|
|
97
|
-
#
|
98
|
+
# Run comparison utility.
|
98
99
|
def compare!
|
99
100
|
@compare = true
|
100
101
|
end
|
@@ -105,9 +106,27 @@ module Benchmark
|
|
105
106
|
!!@held_path
|
106
107
|
end
|
107
108
|
|
108
|
-
#
|
109
|
+
# Hold after each iteration.
|
110
|
+
# @param held_path [String] File name to store hold file.
|
109
111
|
def hold!(held_path)
|
110
112
|
@held_path = held_path
|
113
|
+
@run_single = true
|
114
|
+
end
|
115
|
+
|
116
|
+
# Save interim results. Similar to hold, but all reports are run
|
117
|
+
# The report label must change for each invocation.
|
118
|
+
# One way to achieve this is to include the version in the label.
|
119
|
+
# @param held_path [String] File name to store hold file.
|
120
|
+
def save!(held_path)
|
121
|
+
@held_path = held_path
|
122
|
+
@run_single = false
|
123
|
+
end
|
124
|
+
|
125
|
+
# Return true if items are to be run one at a time.
|
126
|
+
# For the traditional hold, this is true
|
127
|
+
# @return [Boolean] Run just a single item?
|
128
|
+
def run_single?
|
129
|
+
@run_single
|
111
130
|
end
|
112
131
|
|
113
132
|
# Return true if job needs to generate json.
|
@@ -116,7 +135,7 @@ module Benchmark
|
|
116
135
|
!!@json_path
|
117
136
|
end
|
118
137
|
|
119
|
-
#
|
138
|
+
# Generate json to given path, defaults to "data.json".
|
120
139
|
def json!(path="data.json")
|
121
140
|
@json_path = path
|
122
141
|
end
|
@@ -166,84 +185,108 @@ module Benchmark
|
|
166
185
|
def iterations_per_sec cycles, time_us
|
167
186
|
MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
|
168
187
|
end
|
169
|
-
|
170
|
-
def held_results?
|
171
|
-
File.exist?(@held_path)
|
172
|
-
end
|
173
|
-
|
188
|
+
|
174
189
|
def load_held_results
|
190
|
+
return unless @held_path && File.exist?(@held_path)
|
191
|
+
require "json"
|
192
|
+
@held_results = {}
|
193
|
+
JSON.load(IO.read(@held_path)).each do |result|
|
194
|
+
@held_results[result['item']] = result
|
195
|
+
create_report(result['item'], result['measured_us'], result['iter'],
|
196
|
+
create_stats(result['samples']), result['cycles'])
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def save_held_results
|
201
|
+
return unless @held_path
|
175
202
|
require "json"
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
203
|
+
data = full_report.entries.map { |e|
|
204
|
+
{
|
205
|
+
'item' => e.label,
|
206
|
+
'measured_us' => e.microseconds,
|
207
|
+
'iter' => e.iterations,
|
208
|
+
'samples' => e.samples,
|
209
|
+
'cycles' => e.measurement_cycle
|
210
|
+
}
|
211
|
+
}
|
212
|
+
IO.write(@held_path, JSON.generate(data) << "\n")
|
213
|
+
end
|
214
|
+
|
215
|
+
def all_results_have_been_run?
|
216
|
+
@full_report.entries.size == @list.size
|
180
217
|
end
|
181
|
-
|
218
|
+
|
219
|
+
def clear_held_results
|
220
|
+
File.delete @held_path if File.exist?(@held_path)
|
221
|
+
end
|
222
|
+
|
182
223
|
def run
|
183
|
-
@
|
184
|
-
|
185
|
-
|
224
|
+
if @warmup && @warmup != 0 then
|
225
|
+
@stdout.start_warming if @stdout
|
226
|
+
@iterations.times do
|
227
|
+
run_warmup
|
228
|
+
end
|
186
229
|
end
|
187
|
-
|
230
|
+
|
188
231
|
@stdout.start_running if @stdout
|
189
|
-
|
190
|
-
held = nil
|
191
|
-
|
232
|
+
|
192
233
|
@iterations.times do |n|
|
193
|
-
|
234
|
+
run_benchmark
|
194
235
|
end
|
195
236
|
|
196
237
|
@stdout.footer if @stdout
|
197
|
-
|
198
|
-
if held
|
199
|
-
puts
|
200
|
-
puts 'Pausing here -- run Ruby again to measure the next benchmark...'
|
201
|
-
end
|
202
238
|
end
|
203
239
|
|
204
240
|
# Run warmup.
|
205
241
|
def run_warmup
|
206
242
|
@list.each do |item|
|
207
|
-
next if
|
208
|
-
|
243
|
+
next if run_single? && @held_results && @held_results.key?(item.label)
|
244
|
+
|
209
245
|
@suite.warming item.label, @warmup if @suite
|
210
246
|
@stdout.warming item.label, @warmup if @stdout
|
211
247
|
|
212
248
|
Timing.clean_env
|
213
249
|
|
250
|
+
# Run for up to half of the configured warmup time with an increasing
|
251
|
+
# number of cycles to reduce overhead and improve accuracy.
|
252
|
+
# This also avoids running with a constant number of cycles, which a
|
253
|
+
# JIT might speculate on and then have to recompile in #run_benchmark.
|
214
254
|
before = Timing.now
|
215
|
-
target = Timing.add_second before, @warmup
|
255
|
+
target = Timing.add_second before, @warmup / 2.0
|
216
256
|
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
257
|
+
cycles = 1
|
258
|
+
warmup_iter = 1
|
259
|
+
warmup_time_us = 0.0
|
260
|
+
while Timing.now + warmup_time_us * 2 < target
|
261
|
+
t0 = Timing.now
|
262
|
+
item.call_times cycles
|
263
|
+
t1 = Timing.now
|
264
|
+
warmup_iter = cycles
|
265
|
+
warmup_time_us = Timing.time_us(t0, t1)
|
266
|
+
cycles *= 2
|
222
267
|
end
|
223
268
|
|
224
|
-
|
269
|
+
cycles = cycles_per_100ms warmup_time_us, warmup_iter
|
270
|
+
@timing[item] = cycles
|
225
271
|
|
226
|
-
|
227
|
-
|
228
|
-
|
272
|
+
# Run for the remaining of warmup in a similar way as #run_benchmark.
|
273
|
+
target = Timing.add_second before, @warmup
|
274
|
+
while Timing.now + MICROSECONDS_PER_100MS < target
|
275
|
+
item.call_times cycles
|
276
|
+
end
|
229
277
|
|
230
278
|
@stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
|
231
279
|
@suite.warmup_stats warmup_time_us, @timing[item] if @suite
|
232
|
-
|
233
|
-
break if
|
280
|
+
|
281
|
+
break if run_single?
|
234
282
|
end
|
235
283
|
end
|
236
284
|
|
237
285
|
# Run calculation.
|
238
286
|
def run_benchmark
|
239
287
|
@list.each do |item|
|
240
|
-
if
|
241
|
-
|
242
|
-
create_report(item.label, result['measured_us'], result['iter'],
|
243
|
-
create_stats(result['samples']), result['cycles'])
|
244
|
-
next
|
245
|
-
end
|
246
|
-
|
288
|
+
next if run_single? && @held_results && @held_results.key?(item.label)
|
289
|
+
|
247
290
|
@suite.running item.label, @time if @suite
|
248
291
|
@stdout.running item.label, @time if @stdout
|
249
292
|
|
@@ -257,7 +300,7 @@ module Benchmark
|
|
257
300
|
cycles = @timing[item]
|
258
301
|
|
259
302
|
target = Timing.add_second Timing.now, @time
|
260
|
-
|
303
|
+
|
261
304
|
while (before = Timing.now) < target
|
262
305
|
item.call_times cycles
|
263
306
|
after = Timing.now
|
@@ -288,29 +331,9 @@ module Benchmark
|
|
288
331
|
|
289
332
|
@stdout.add_report rep, caller(1).first if @stdout
|
290
333
|
@suite.add_report rep, caller(1).first if @suite
|
291
|
-
|
292
|
-
if
|
293
|
-
File.open @held_path, "a" do |f|
|
294
|
-
require "json"
|
295
|
-
f.write JSON.generate({
|
296
|
-
:item => item.label,
|
297
|
-
:measured_us => measured_us,
|
298
|
-
:iter => iter,
|
299
|
-
:samples => samples,
|
300
|
-
:cycles => cycles
|
301
|
-
})
|
302
|
-
f.write "\n"
|
303
|
-
end
|
304
|
-
|
305
|
-
return true
|
306
|
-
end
|
307
|
-
end
|
308
|
-
|
309
|
-
if hold? && @full_report.entries.size == @list.size
|
310
|
-
File.delete @held_path if File.exist?(@held_path)
|
334
|
+
|
335
|
+
break if run_single?
|
311
336
|
end
|
312
|
-
|
313
|
-
false
|
314
337
|
end
|
315
338
|
|
316
339
|
def create_stats(samples)
|
@@ -2,10 +2,14 @@ module Benchmark
|
|
2
2
|
module IPS
|
3
3
|
class Job
|
4
4
|
class StdoutReport
|
5
|
+
def initialize
|
6
|
+
@last_item = nil
|
7
|
+
end
|
8
|
+
|
5
9
|
def start_warming
|
6
10
|
$stdout.puts "Warming up --------------------------------------"
|
7
11
|
end
|
8
|
-
|
12
|
+
|
9
13
|
def start_running
|
10
14
|
$stdout.puts "Calculating -------------------------------------"
|
11
15
|
end
|
@@ -31,6 +35,7 @@ module Benchmark
|
|
31
35
|
end
|
32
36
|
|
33
37
|
def footer
|
38
|
+
return unless @last_item
|
34
39
|
footer = @last_item.stats.footer
|
35
40
|
$stdout.puts footer.rjust(40) if footer
|
36
41
|
end
|
data/lib/benchmark/ips/report.rb
CHANGED
@@ -52,6 +52,10 @@ module Benchmark
|
|
52
52
|
@stats.error
|
53
53
|
end
|
54
54
|
|
55
|
+
def samples
|
56
|
+
@stats.samples
|
57
|
+
end
|
58
|
+
|
55
59
|
# Number of Cycles.
|
56
60
|
# @return [Integer] number of cycles.
|
57
61
|
attr_reader :measurement_cycle
|
@@ -72,7 +76,7 @@ module Benchmark
|
|
72
76
|
# Return entry's standard deviation of iteration per second in percentage.
|
73
77
|
# @return [Float] +@ips_sd+ in percentage.
|
74
78
|
def error_percentage
|
75
|
-
|
79
|
+
@stats.error_percentage
|
76
80
|
end
|
77
81
|
|
78
82
|
alias_method :runtime, :seconds
|
@@ -84,7 +88,7 @@ module Benchmark
|
|
84
88
|
def body
|
85
89
|
case Benchmark::IPS.options[:format]
|
86
90
|
when :human
|
87
|
-
left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
|
91
|
+
left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
|
88
92
|
iters = Helpers.scale(@iterations)
|
89
93
|
|
90
94
|
if @show_total_time
|
@@ -93,7 +97,7 @@ module Benchmark
|
|
93
97
|
left.ljust(20) + (" - %s" % iters)
|
94
98
|
end
|
95
99
|
else
|
96
|
-
left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
|
100
|
+
left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
|
97
101
|
|
98
102
|
if @show_total_time
|
99
103
|
left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
|
@@ -3,27 +3,30 @@ module Benchmark
|
|
3
3
|
module Stats
|
4
4
|
|
5
5
|
class Bootstrap
|
6
|
-
|
7
|
-
attr_reader :data
|
6
|
+
include StatsMetric
|
7
|
+
attr_reader :data, :error, :samples
|
8
8
|
|
9
9
|
def initialize(samples, confidence)
|
10
10
|
dependencies
|
11
11
|
@iterations = 10_000
|
12
12
|
@confidence = (confidence / 100.0).to_s
|
13
|
+
@samples = samples
|
13
14
|
@data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
|
14
15
|
interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
|
15
16
|
@median = interval.median
|
16
17
|
@error = interval.error
|
17
18
|
end
|
18
19
|
|
20
|
+
# Average stat value
|
21
|
+
# @return [Float] central_tendency
|
19
22
|
def central_tendency
|
20
23
|
@median
|
21
24
|
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
26
|
+
# Determines how much slower this stat is than the baseline stat
|
27
|
+
# if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
|
28
|
+
# @param baseline [SD|Bootstrap] faster baseline
|
29
|
+
# @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
|
27
30
|
def slowdown(baseline)
|
28
31
|
low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
|
29
32
|
error = Timing.mean([slowdown - low, high - slowdown])
|
@@ -1,33 +1,41 @@
|
|
1
1
|
module Benchmark
|
2
2
|
module IPS
|
3
3
|
module Stats
|
4
|
-
|
4
|
+
|
5
5
|
class SD
|
6
|
-
|
6
|
+
include StatsMetric
|
7
|
+
attr_reader :error, :samples
|
8
|
+
|
7
9
|
def initialize(samples)
|
10
|
+
@samples = samples
|
8
11
|
@mean = Timing.mean(samples)
|
9
12
|
@error = Timing.stddev(samples, @mean).round
|
10
13
|
end
|
11
|
-
|
14
|
+
|
15
|
+
# Average stat value
|
16
|
+
# @return [Float] central_tendency
|
12
17
|
def central_tendency
|
13
18
|
@mean
|
14
19
|
end
|
15
|
-
|
16
|
-
def error
|
17
|
-
@error
|
18
|
-
end
|
19
20
|
|
21
|
+
# Determines how much slower this stat is than the baseline stat
|
22
|
+
# if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
|
23
|
+
# @param baseline [SD|Bootstrap] faster baseline
|
24
|
+
# @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
|
20
25
|
def slowdown(baseline)
|
21
|
-
|
22
|
-
|
26
|
+
if baseline.central_tendency > central_tendency
|
27
|
+
[baseline.central_tendency.to_f / central_tendency, 0]
|
28
|
+
else
|
29
|
+
[central_tendency.to_f / baseline.central_tendency, 0]
|
30
|
+
end
|
23
31
|
end
|
24
32
|
|
25
33
|
def footer
|
26
34
|
nil
|
27
35
|
end
|
28
|
-
|
36
|
+
|
29
37
|
end
|
30
|
-
|
38
|
+
|
31
39
|
end
|
32
40
|
end
|
33
41
|
end
|
data/test/test_benchmark_ips.rb
CHANGED
@@ -20,6 +20,19 @@ class TestBenchmarkIPS < Minitest::Test
|
|
20
20
|
assert $stdout.string.size > 0
|
21
21
|
end
|
22
22
|
|
23
|
+
def test_warmup0
|
24
|
+
$stdout = @old_stdout
|
25
|
+
|
26
|
+
out, err = capture_io do
|
27
|
+
Benchmark.ips(:time => 1, :warmup => 0, :quiet => false) do |x|
|
28
|
+
x.report("sleep 0.25") { sleep(0.25) }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
refute_match(/Warming up -+/, out)
|
33
|
+
assert_empty err
|
34
|
+
end
|
35
|
+
|
23
36
|
def test_output
|
24
37
|
Benchmark.ips(1) do |x|
|
25
38
|
x.report("operation") { 100 * 100 }
|
@@ -140,6 +153,17 @@ class TestBenchmarkIPS < Minitest::Test
|
|
140
153
|
assert all_data[0][:stddev]
|
141
154
|
end
|
142
155
|
|
156
|
+
def test_ips_empty
|
157
|
+
report = Benchmark.ips do |_x|
|
158
|
+
|
159
|
+
end
|
160
|
+
|
161
|
+
all_data = report.data
|
162
|
+
|
163
|
+
assert all_data
|
164
|
+
assert_equal [], all_data
|
165
|
+
end
|
166
|
+
|
143
167
|
def test_json_output
|
144
168
|
json_file = Tempfile.new("data.json")
|
145
169
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark-ips
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Phoenix
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-05-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: minitest
|
@@ -16,42 +16,48 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '5.
|
19
|
+
version: '5.14'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '5.
|
26
|
+
version: '5.14'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rdoc
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '4.0'
|
34
|
+
- - "<"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '7'
|
34
37
|
type: :development
|
35
38
|
prerelease: false
|
36
39
|
version_requirements: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
|
-
- - "
|
41
|
+
- - ">="
|
39
42
|
- !ruby/object:Gem::Version
|
40
43
|
version: '4.0'
|
44
|
+
- - "<"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '7'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: hoe
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version: '3.
|
53
|
+
version: '3.22'
|
48
54
|
type: :development
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
51
57
|
requirements:
|
52
58
|
- - "~>"
|
53
59
|
- !ruby/object:Gem::Version
|
54
|
-
version: '3.
|
60
|
+
version: '3.22'
|
55
61
|
description: An iterations per second enhancement to Benchmark.
|
56
62
|
email:
|
57
63
|
- evan@phx.io
|
@@ -63,7 +69,6 @@ extra_rdoc_files:
|
|
63
69
|
- README.md
|
64
70
|
files:
|
65
71
|
- ".autotest"
|
66
|
-
- Gemfile.lock
|
67
72
|
- History.txt
|
68
73
|
- Manifest.txt
|
69
74
|
- README.md
|
@@ -82,7 +87,8 @@ files:
|
|
82
87
|
homepage: https://github.com/evanphx/benchmark-ips
|
83
88
|
licenses:
|
84
89
|
- MIT
|
85
|
-
metadata:
|
90
|
+
metadata:
|
91
|
+
homepage_uri: https://github.com/evanphx/benchmark-ips
|
86
92
|
post_install_message:
|
87
93
|
rdoc_options:
|
88
94
|
- "--main"
|
@@ -100,8 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
106
|
- !ruby/object:Gem::Version
|
101
107
|
version: '0'
|
102
108
|
requirements: []
|
103
|
-
|
104
|
-
rubygems_version: 2.5.1
|
109
|
+
rubygems_version: 3.0.3
|
105
110
|
signing_key:
|
106
111
|
specification_version: 4
|
107
112
|
summary: An iterations per second enhancement to Benchmark.
|
data/Gemfile.lock
DELETED