benchmark-ips 2.7.1 → 2.8.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/History.txt +38 -0
- data/Manifest.txt +1 -1
- data/README.md +10 -4
- data/lib/benchmark/compare.rb +5 -9
- data/lib/benchmark/ips.rb +75 -3
- data/lib/benchmark/ips/job.rb +99 -71
- data/lib/benchmark/ips/job/entry.rb +33 -15
- data/lib/benchmark/ips/job/stdout_report.rb +6 -1
- data/lib/benchmark/ips/report.rb +19 -3
- data/lib/benchmark/ips/stats/bootstrap.rb +9 -6
- data/lib/benchmark/ips/stats/sd.rb +19 -11
- data/lib/benchmark/ips/stats/stats_metric.rb +21 -0
- data/test/test_benchmark_ips.rb +44 -6
- metadata +21 -15
- data/Gemfile.lock +0 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 574d0cb84279e02316df55c8d96f396b2a10301b15639ede54198eb278fd6ba2
|
4
|
+
data.tar.gz: d4ddabd9c093fd2c9559870818543d105c6fb2bbf3d38ad397b3bdf0df691397
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 701868e1d0e33101daaf7e5005245b327dbf3b3d02fb19a8e89eb39563540b5ca5b1ff91c4e3cd4bd9dd54f97537ca48e668608742ee0569a13336818ddfef23
|
7
|
+
data.tar.gz: a032682bd26691ad5d4a837ba956b9acf50e45c69036e07bdda347791f78bf388cb3339873fd6d5ca9cbf6edeec5090c15ae6fa56a712f6d0f0340729ce8e792
|
data/History.txt
CHANGED
@@ -1,3 +1,41 @@
|
|
1
|
+
=== 2.8.4 / 2020-12-03
|
2
|
+
|
3
|
+
* Bug fix
|
4
|
+
* Fixed hold! when results file does not exist.
|
5
|
+
|
6
|
+
=== 2.8.3 / 2020-08-28
|
7
|
+
|
8
|
+
* Bug fix
|
9
|
+
* Fixed inaccuracy caused by integer overflows.
|
10
|
+
|
11
|
+
=== 2.8.2 / 2020-05-04
|
12
|
+
|
13
|
+
* Bug fix
|
14
|
+
* Fixed problems with Manifest.txt.
|
15
|
+
* Empty interim results files are ignored.
|
16
|
+
|
17
|
+
=== 2.8.0 / 2020-05-01
|
18
|
+
|
19
|
+
* Feature
|
20
|
+
* Allow running with empty ips block.
|
21
|
+
* Added save! method for saving interim results.
|
22
|
+
* Run more than just 1 cycle during warmup to reduce overhead.
|
23
|
+
* Optimized Job::Entry hot-path for fairer results on JRuby/TruffleRuby.
|
24
|
+
|
25
|
+
* Bug fix
|
26
|
+
* Removed the warmup section if set to 0.
|
27
|
+
* Added some RDoc docs.
|
28
|
+
* Added some examples in examples/
|
29
|
+
|
30
|
+
=== 2.7.2 / 2016-08-18
|
31
|
+
|
32
|
+
* 1 bug fix:
|
33
|
+
* Restore old accessors. Fixes #76
|
34
|
+
|
35
|
+
=== 2.7.1 / 2016-08-08
|
36
|
+
|
37
|
+
Add missing files
|
38
|
+
|
1
39
|
=== 2.7.0 / 2016-08-05
|
2
40
|
|
3
41
|
* 1 minor features:
|
data/Manifest.txt
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
.autotest
|
2
|
-
Gemfile.lock
|
3
2
|
History.txt
|
4
3
|
Manifest.txt
|
5
4
|
README.md
|
@@ -13,5 +12,6 @@ lib/benchmark/ips/report.rb
|
|
13
12
|
lib/benchmark/ips/share.rb
|
14
13
|
lib/benchmark/ips/stats/bootstrap.rb
|
15
14
|
lib/benchmark/ips/stats/sd.rb
|
15
|
+
lib/benchmark/ips/stats/stats_metric.rb
|
16
16
|
lib/benchmark/timing.rb
|
17
17
|
test/test_benchmark_ips.rb
|
data/README.md
CHANGED
@@ -1,13 +1,14 @@
|
|
1
|
+
# benchmark-ips
|
2
|
+
|
3
|
+
* rdoc :: http://rubydoc.info/gems/benchmark-ips
|
4
|
+
* home :: https://github.com/evanphx/benchmark-ips
|
5
|
+
|
1
6
|
[![Gem Version](https://badge.fury.io/rb/benchmark-ips.svg)](http://badge.fury.io/rb/benchmark-ips)
|
2
7
|
[![Build Status](https://secure.travis-ci.org/evanphx/benchmark-ips.svg)](http://travis-ci.org/evanphx/benchmark-ips)
|
3
8
|
[![Inline docs](http://inch-ci.org/github/evanphx/benchmark-ips.svg)](http://inch-ci.org/github/evanphx/benchmark-ips)
|
4
9
|
|
5
|
-
# benchmark-ips
|
6
|
-
|
7
10
|
* https://github.com/evanphx/benchmark-ips
|
8
11
|
|
9
|
-
* [documentation](http://rubydoc.info/gems/benchmark-ips)
|
10
|
-
|
11
12
|
## DESCRIPTION:
|
12
13
|
|
13
14
|
An iterations per second enhancement to Benchmark.
|
@@ -155,6 +156,11 @@ This will run only one benchmarks each time you run the command, storing
|
|
155
156
|
results in the specified file. The file is deleted when all results have been
|
156
157
|
gathered and the report is shown.
|
157
158
|
|
159
|
+
Alternatively, if you prefer a different approach, the `save!` command is
|
160
|
+
available. Examples for [hold!](examples/hold.rb) and [save!](examples/save.rb) are available in
|
161
|
+
the `examples/` directory.
|
162
|
+
|
163
|
+
|
158
164
|
### Multiple iterations
|
159
165
|
|
160
166
|
In some cases you may want to run multiple iterations of the warmup and
|
data/lib/benchmark/compare.rb
CHANGED
@@ -40,18 +40,14 @@ module Benchmark
|
|
40
40
|
|
41
41
|
$stdout.puts "\nComparison:"
|
42
42
|
|
43
|
-
$stdout.printf "%20s: %10.1f i/s\n", best.label, best.stats.central_tendency
|
43
|
+
$stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
|
44
44
|
|
45
45
|
sorted.each do |report|
|
46
46
|
name = report.label.to_s
|
47
|
-
|
47
|
+
|
48
48
|
$stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
|
49
|
-
|
50
|
-
|
51
|
-
report_high = report.stats.central_tendency + report.stats.error
|
52
|
-
overlaps = report_high > best_low
|
53
|
-
|
54
|
-
if overlaps
|
49
|
+
|
50
|
+
if report.stats.overlaps?(best.stats)
|
55
51
|
$stdout.print "same-ish: difference falls within error"
|
56
52
|
else
|
57
53
|
slowdown, error = report.stats.slowdown(best.stats)
|
@@ -61,7 +57,7 @@ module Benchmark
|
|
61
57
|
end
|
62
58
|
$stdout.print " slower"
|
63
59
|
end
|
64
|
-
|
60
|
+
|
65
61
|
$stdout.puts
|
66
62
|
end
|
67
63
|
|
data/lib/benchmark/ips.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'benchmark/timing'
|
3
3
|
require 'benchmark/compare'
|
4
|
+
require 'benchmark/ips/stats/stats_metric'
|
4
5
|
require 'benchmark/ips/stats/sd'
|
5
6
|
require 'benchmark/ips/stats/bootstrap'
|
6
7
|
require 'benchmark/ips/report'
|
@@ -15,10 +16,10 @@ module Benchmark
|
|
15
16
|
module IPS
|
16
17
|
|
17
18
|
# Benchmark-ips Gem version.
|
18
|
-
VERSION = "2.
|
19
|
+
VERSION = "2.8.4"
|
19
20
|
|
20
21
|
# CODENAME of current version.
|
21
|
-
CODENAME = "
|
22
|
+
CODENAME = "Tardy Turtle"
|
22
23
|
|
23
24
|
# Measure code in block, each code's benchmarked result will display in
|
24
25
|
# iteration per second with standard deviation in given time.
|
@@ -54,10 +55,17 @@ module Benchmark
|
|
54
55
|
|
55
56
|
yield job
|
56
57
|
|
57
|
-
job.load_held_results
|
58
|
+
job.load_held_results
|
58
59
|
|
59
60
|
job.run
|
60
61
|
|
62
|
+
if job.run_single? && job.all_results_have_been_run?
|
63
|
+
job.clear_held_results
|
64
|
+
else
|
65
|
+
job.save_held_results
|
66
|
+
puts '', 'Pausing here -- run Ruby again to measure the next benchmark...' if job.run_single?
|
67
|
+
end
|
68
|
+
|
61
69
|
$stdout.sync = sync
|
62
70
|
job.run_comparison
|
63
71
|
job.generate_json
|
@@ -102,4 +110,68 @@ module Benchmark
|
|
102
110
|
end
|
103
111
|
|
104
112
|
extend Benchmark::IPS # make ips available as module-level method
|
113
|
+
|
114
|
+
##
|
115
|
+
# :singleton-method: ips
|
116
|
+
#
|
117
|
+
# require 'benchmark/ips'
|
118
|
+
#
|
119
|
+
# Benchmark.ips do |x|
|
120
|
+
# # Configure the number of seconds used during
|
121
|
+
# # the warmup phase (default 2) and calculation phase (default 5)
|
122
|
+
# x.config(:time => 5, :warmup => 2)
|
123
|
+
#
|
124
|
+
# # These parameters can also be configured this way
|
125
|
+
# x.time = 5
|
126
|
+
# x.warmup = 2
|
127
|
+
#
|
128
|
+
# # Typical mode, runs the block as many times as it can
|
129
|
+
# x.report("addition") { 1 + 2 }
|
130
|
+
#
|
131
|
+
# # To reduce overhead, the number of iterations is passed in
|
132
|
+
# # and the block must run the code the specific number of times.
|
133
|
+
# # Used for when the workload is very small and any overhead
|
134
|
+
# # introduces incorrectable errors.
|
135
|
+
# x.report("addition2") do |times|
|
136
|
+
# i = 0
|
137
|
+
# while i < times
|
138
|
+
# 1 + 2
|
139
|
+
# i += 1
|
140
|
+
# end
|
141
|
+
# end
|
142
|
+
#
|
143
|
+
# # To reduce overhead even more, grafts the code given into
|
144
|
+
# # the loop that performs the iterations internally to reduce
|
145
|
+
# # overhead. Typically not needed, use the |times| form instead.
|
146
|
+
# x.report("addition3", "1 + 2")
|
147
|
+
#
|
148
|
+
# # Really long labels should be formatted correctly
|
149
|
+
# x.report("addition-test-long-label") { 1 + 2 }
|
150
|
+
#
|
151
|
+
# # Compare the iterations per second of the various reports!
|
152
|
+
# x.compare!
|
153
|
+
# end
|
154
|
+
#
|
155
|
+
# This will generate the following report:
|
156
|
+
#
|
157
|
+
# Calculating -------------------------------------
|
158
|
+
# addition 71.254k i/100ms
|
159
|
+
# addition2 68.658k i/100ms
|
160
|
+
# addition3 83.079k i/100ms
|
161
|
+
# addition-test-long-label
|
162
|
+
# 70.129k i/100ms
|
163
|
+
# -------------------------------------------------
|
164
|
+
# addition 4.955M (± 8.7%) i/s - 24.155M
|
165
|
+
# addition2 24.011M (± 9.5%) i/s - 114.246M
|
166
|
+
# addition3 23.958M (±10.1%) i/s - 115.064M
|
167
|
+
# addition-test-long-label
|
168
|
+
# 5.014M (± 9.1%) i/s - 24.545M
|
169
|
+
#
|
170
|
+
# Comparison:
|
171
|
+
# addition2: 24011974.8 i/s
|
172
|
+
# addition3: 23958619.8 i/s - 1.00x slower
|
173
|
+
# addition-test-long-label: 5014756.0 i/s - 4.79x slower
|
174
|
+
# addition: 4955278.9 i/s - 4.85x slower
|
175
|
+
#
|
176
|
+
# See also Benchmark::IPS
|
105
177
|
end
|
data/lib/benchmark/ips/job.rb
CHANGED
@@ -9,6 +9,7 @@ module Benchmark
|
|
9
9
|
# The percentage of the expected runtime to allow
|
10
10
|
# before reporting a weird runtime
|
11
11
|
MAX_TIME_SKEW = 0.05
|
12
|
+
POW_2_30 = 1 << 30
|
12
13
|
|
13
14
|
# Two-element arrays, consisting of label and block pairs.
|
14
15
|
# @return [Array<Entry>] list of entries
|
@@ -58,11 +59,12 @@ module Benchmark
|
|
58
59
|
@stdout = opts[:quiet] ? nil : StdoutReport.new
|
59
60
|
@list = []
|
60
61
|
@compare = false
|
62
|
+
@run_single = false
|
61
63
|
@json_path = false
|
62
64
|
@held_path = nil
|
63
65
|
@held_results = nil
|
64
66
|
|
65
|
-
@timing =
|
67
|
+
@timing = Hash.new 1 # default to 1 in case warmup isn't run
|
66
68
|
@full_report = Report.new
|
67
69
|
|
68
70
|
# Default warmup and calculation time in seconds.
|
@@ -94,7 +96,7 @@ module Benchmark
|
|
94
96
|
@compare
|
95
97
|
end
|
96
98
|
|
97
|
-
#
|
99
|
+
# Run comparison utility.
|
98
100
|
def compare!
|
99
101
|
@compare = true
|
100
102
|
end
|
@@ -105,9 +107,27 @@ module Benchmark
|
|
105
107
|
!!@held_path
|
106
108
|
end
|
107
109
|
|
108
|
-
#
|
110
|
+
# Hold after each iteration.
|
111
|
+
# @param held_path [String] File name to store hold file.
|
109
112
|
def hold!(held_path)
|
110
113
|
@held_path = held_path
|
114
|
+
@run_single = true
|
115
|
+
end
|
116
|
+
|
117
|
+
# Save interim results. Similar to hold, but all reports are run
|
118
|
+
# The report label must change for each invocation.
|
119
|
+
# One way to achieve this is to include the version in the label.
|
120
|
+
# @param held_path [String] File name to store hold file.
|
121
|
+
def save!(held_path)
|
122
|
+
@held_path = held_path
|
123
|
+
@run_single = false
|
124
|
+
end
|
125
|
+
|
126
|
+
# Return true if items are to be run one at a time.
|
127
|
+
# For the traditional hold, this is true
|
128
|
+
# @return [Boolean] Run just a single item?
|
129
|
+
def run_single?
|
130
|
+
@run_single
|
111
131
|
end
|
112
132
|
|
113
133
|
# Return true if job needs to generate json.
|
@@ -116,7 +136,7 @@ module Benchmark
|
|
116
136
|
!!@json_path
|
117
137
|
end
|
118
138
|
|
119
|
-
#
|
139
|
+
# Generate json to given path, defaults to "data.json".
|
120
140
|
def json!(path="data.json")
|
121
141
|
@json_path = path
|
122
142
|
end
|
@@ -166,84 +186,112 @@ module Benchmark
|
|
166
186
|
def iterations_per_sec cycles, time_us
|
167
187
|
MICROSECONDS_PER_SECOND * (cycles.to_f / time_us.to_f)
|
168
188
|
end
|
169
|
-
|
170
|
-
def held_results?
|
171
|
-
File.exist?(@held_path)
|
172
|
-
end
|
173
|
-
|
189
|
+
|
174
190
|
def load_held_results
|
191
|
+
return unless @held_path && File.exist?(@held_path) && !File.zero?(@held_path)
|
192
|
+
require "json"
|
193
|
+
@held_results = {}
|
194
|
+
JSON.load(IO.read(@held_path)).each do |result|
|
195
|
+
@held_results[result['item']] = result
|
196
|
+
create_report(result['item'], result['measured_us'], result['iter'],
|
197
|
+
create_stats(result['samples']), result['cycles'])
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def save_held_results
|
202
|
+
return unless @held_path
|
175
203
|
require "json"
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
204
|
+
data = full_report.entries.map { |e|
|
205
|
+
{
|
206
|
+
'item' => e.label,
|
207
|
+
'measured_us' => e.microseconds,
|
208
|
+
'iter' => e.iterations,
|
209
|
+
'samples' => e.samples,
|
210
|
+
'cycles' => e.measurement_cycle
|
211
|
+
}
|
212
|
+
}
|
213
|
+
IO.write(@held_path, JSON.generate(data) << "\n")
|
214
|
+
end
|
215
|
+
|
216
|
+
def all_results_have_been_run?
|
217
|
+
@full_report.entries.size == @list.size
|
180
218
|
end
|
181
|
-
|
219
|
+
|
220
|
+
def clear_held_results
|
221
|
+
File.delete @held_path if File.exist?(@held_path)
|
222
|
+
end
|
223
|
+
|
182
224
|
def run
|
183
|
-
@
|
184
|
-
|
185
|
-
|
225
|
+
if @warmup && @warmup != 0 then
|
226
|
+
@stdout.start_warming if @stdout
|
227
|
+
@iterations.times do
|
228
|
+
run_warmup
|
229
|
+
end
|
186
230
|
end
|
187
|
-
|
231
|
+
|
188
232
|
@stdout.start_running if @stdout
|
189
|
-
|
190
|
-
held = nil
|
191
|
-
|
233
|
+
|
192
234
|
@iterations.times do |n|
|
193
|
-
|
235
|
+
run_benchmark
|
194
236
|
end
|
195
237
|
|
196
238
|
@stdout.footer if @stdout
|
197
|
-
|
198
|
-
if held
|
199
|
-
puts
|
200
|
-
puts 'Pausing here -- run Ruby again to measure the next benchmark...'
|
201
|
-
end
|
202
239
|
end
|
203
240
|
|
204
241
|
# Run warmup.
|
205
242
|
def run_warmup
|
206
243
|
@list.each do |item|
|
207
|
-
next if
|
208
|
-
|
244
|
+
next if run_single? && @held_results && @held_results.key?(item.label)
|
245
|
+
|
209
246
|
@suite.warming item.label, @warmup if @suite
|
210
247
|
@stdout.warming item.label, @warmup if @stdout
|
211
248
|
|
212
249
|
Timing.clean_env
|
213
250
|
|
251
|
+
# Run for up to half of the configured warmup time with an increasing
|
252
|
+
# number of cycles to reduce overhead and improve accuracy.
|
253
|
+
# This also avoids running with a constant number of cycles, which a
|
254
|
+
# JIT might speculate on and then have to recompile in #run_benchmark.
|
214
255
|
before = Timing.now
|
215
|
-
target = Timing.add_second before, @warmup
|
256
|
+
target = Timing.add_second before, @warmup / 2.0
|
216
257
|
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
258
|
+
cycles = 1
|
259
|
+
warmup_iter = 1
|
260
|
+
warmup_time_us = 0.0
|
261
|
+
while Timing.now + warmup_time_us * 2 < target
|
262
|
+
t0 = Timing.now
|
263
|
+
item.call_times cycles
|
264
|
+
t1 = Timing.now
|
265
|
+
warmup_iter = cycles
|
266
|
+
warmup_time_us = Timing.time_us(t0, t1)
|
267
|
+
|
268
|
+
# If the number of cycles would go outside the 32-bit signed integers range
|
269
|
+
# then exit the loop to avoid overflows and start the 100ms warmup runs
|
270
|
+
break if cycles >= POW_2_30
|
271
|
+
cycles *= 2
|
222
272
|
end
|
223
273
|
|
224
|
-
|
274
|
+
cycles = cycles_per_100ms warmup_time_us, warmup_iter
|
275
|
+
@timing[item] = cycles
|
225
276
|
|
226
|
-
|
227
|
-
|
228
|
-
|
277
|
+
# Run for the remaining of warmup in a similar way as #run_benchmark.
|
278
|
+
target = Timing.add_second before, @warmup
|
279
|
+
while Timing.now + MICROSECONDS_PER_100MS < target
|
280
|
+
item.call_times cycles
|
281
|
+
end
|
229
282
|
|
230
283
|
@stdout.warmup_stats warmup_time_us, @timing[item] if @stdout
|
231
284
|
@suite.warmup_stats warmup_time_us, @timing[item] if @suite
|
232
|
-
|
233
|
-
break if
|
285
|
+
|
286
|
+
break if run_single?
|
234
287
|
end
|
235
288
|
end
|
236
289
|
|
237
290
|
# Run calculation.
|
238
291
|
def run_benchmark
|
239
292
|
@list.each do |item|
|
240
|
-
if
|
241
|
-
|
242
|
-
create_report(item.label, result['measured_us'], result['iter'],
|
243
|
-
create_stats(result['samples']), result['cycles'])
|
244
|
-
next
|
245
|
-
end
|
246
|
-
|
293
|
+
next if run_single? && @held_results && @held_results.key?(item.label)
|
294
|
+
|
247
295
|
@suite.running item.label, @time if @suite
|
248
296
|
@stdout.running item.label, @time if @stdout
|
249
297
|
|
@@ -257,7 +305,7 @@ module Benchmark
|
|
257
305
|
cycles = @timing[item]
|
258
306
|
|
259
307
|
target = Timing.add_second Timing.now, @time
|
260
|
-
|
308
|
+
|
261
309
|
while (before = Timing.now) < target
|
262
310
|
item.call_times cycles
|
263
311
|
after = Timing.now
|
@@ -288,29 +336,9 @@ module Benchmark
|
|
288
336
|
|
289
337
|
@stdout.add_report rep, caller(1).first if @stdout
|
290
338
|
@suite.add_report rep, caller(1).first if @suite
|
291
|
-
|
292
|
-
if
|
293
|
-
File.open @held_path, "a" do |f|
|
294
|
-
require "json"
|
295
|
-
f.write JSON.generate({
|
296
|
-
:item => item.label,
|
297
|
-
:measured_us => measured_us,
|
298
|
-
:iter => iter,
|
299
|
-
:samples => samples,
|
300
|
-
:cycles => cycles
|
301
|
-
})
|
302
|
-
f.write "\n"
|
303
|
-
end
|
304
|
-
|
305
|
-
return true
|
306
|
-
end
|
307
|
-
end
|
308
|
-
|
309
|
-
if hold? && @full_report.entries.size == @list.size
|
310
|
-
File.delete @held_path if File.exist?(@held_path)
|
339
|
+
|
340
|
+
break if run_single?
|
311
341
|
end
|
312
|
-
|
313
|
-
false
|
314
342
|
end
|
315
343
|
|
316
344
|
def create_stats(samples)
|
@@ -11,10 +11,12 @@ module Benchmark
|
|
11
11
|
def initialize(label, action)
|
12
12
|
@label = label
|
13
13
|
|
14
|
+
# We define #call_times on the singleton class of each Entry instance.
|
15
|
+
# That way, there is no polymorphism for `@action.call` inside #call_times.
|
16
|
+
|
14
17
|
if action.kind_of? String
|
15
|
-
|
18
|
+
compile_string action
|
16
19
|
@action = self
|
17
|
-
@as_action = true
|
18
20
|
else
|
19
21
|
unless action.respond_to? :call
|
20
22
|
raise ArgumentError, "invalid action, must respond to #call"
|
@@ -23,12 +25,10 @@ module Benchmark
|
|
23
25
|
@action = action
|
24
26
|
|
25
27
|
if action.respond_to? :arity and action.arity > 0
|
26
|
-
|
28
|
+
compile_block_with_manual_loop
|
27
29
|
else
|
28
|
-
|
30
|
+
compile_block
|
29
31
|
end
|
30
|
-
|
31
|
-
@as_action = false
|
32
32
|
end
|
33
33
|
end
|
34
34
|
|
@@ -40,25 +40,43 @@ module Benchmark
|
|
40
40
|
# @return [String, Proc] Code to be called, could be String / Proc.
|
41
41
|
attr_reader :action
|
42
42
|
|
43
|
-
# Call action by given times
|
43
|
+
# Call action by given times.
|
44
44
|
# @param times [Integer] Times to call +@action+.
|
45
45
|
# @return [Integer] Number of times the +@action+ has been called.
|
46
46
|
def call_times(times)
|
47
|
-
|
47
|
+
raise '#call_times should be redefined per Benchmark::IPS::Job::Entry instance'
|
48
|
+
end
|
48
49
|
|
49
|
-
|
50
|
+
def compile_block
|
51
|
+
m = (class << self; self; end)
|
52
|
+
code = <<-CODE
|
53
|
+
def call_times(times)
|
54
|
+
act = @action
|
50
55
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
+
i = 0
|
57
|
+
while i < times
|
58
|
+
act.call
|
59
|
+
i += 1
|
60
|
+
end
|
61
|
+
end
|
62
|
+
CODE
|
63
|
+
m.class_eval code
|
64
|
+
end
|
65
|
+
|
66
|
+
def compile_block_with_manual_loop
|
67
|
+
m = (class << self; self; end)
|
68
|
+
code = <<-CODE
|
69
|
+
def call_times(times)
|
70
|
+
@action.call(times)
|
71
|
+
end
|
72
|
+
CODE
|
73
|
+
m.class_eval code
|
56
74
|
end
|
57
75
|
|
58
76
|
# Compile code into +call_times+ method.
|
59
77
|
# @param str [String] Code to be compiled.
|
60
78
|
# @return [Symbol] :call_times.
|
61
|
-
def
|
79
|
+
def compile_string(str)
|
62
80
|
m = (class << self; self; end)
|
63
81
|
code = <<-CODE
|
64
82
|
def call_times(__total);
|
@@ -2,10 +2,14 @@ module Benchmark
|
|
2
2
|
module IPS
|
3
3
|
class Job
|
4
4
|
class StdoutReport
|
5
|
+
def initialize
|
6
|
+
@last_item = nil
|
7
|
+
end
|
8
|
+
|
5
9
|
def start_warming
|
6
10
|
$stdout.puts "Warming up --------------------------------------"
|
7
11
|
end
|
8
|
-
|
12
|
+
|
9
13
|
def start_running
|
10
14
|
$stdout.puts "Calculating -------------------------------------"
|
11
15
|
end
|
@@ -31,6 +35,7 @@ module Benchmark
|
|
31
35
|
end
|
32
36
|
|
33
37
|
def footer
|
38
|
+
return unless @last_item
|
34
39
|
footer = @last_item.stats.footer
|
35
40
|
$stdout.puts footer.rjust(40) if footer
|
36
41
|
end
|
data/lib/benchmark/ips/report.rb
CHANGED
@@ -40,6 +40,22 @@ module Benchmark
|
|
40
40
|
# @return [Object] statisical summary.
|
41
41
|
attr_reader :stats
|
42
42
|
|
43
|
+
# LEGACY: Iterations per second.
|
44
|
+
# @return [Float] number of iterations per second.
|
45
|
+
def ips
|
46
|
+
@stats.central_tendency
|
47
|
+
end
|
48
|
+
|
49
|
+
# LEGACY: Standard deviation of iteration per second.
|
50
|
+
# @return [Float] standard deviation of iteration per second.
|
51
|
+
def ips_sd
|
52
|
+
@stats.error
|
53
|
+
end
|
54
|
+
|
55
|
+
def samples
|
56
|
+
@stats.samples
|
57
|
+
end
|
58
|
+
|
43
59
|
# Number of Cycles.
|
44
60
|
# @return [Integer] number of cycles.
|
45
61
|
attr_reader :measurement_cycle
|
@@ -60,7 +76,7 @@ module Benchmark
|
|
60
76
|
# Return entry's standard deviation of iteration per second in percentage.
|
61
77
|
# @return [Float] +@ips_sd+ in percentage.
|
62
78
|
def error_percentage
|
63
|
-
|
79
|
+
@stats.error_percentage
|
64
80
|
end
|
65
81
|
|
66
82
|
alias_method :runtime, :seconds
|
@@ -72,7 +88,7 @@ module Benchmark
|
|
72
88
|
def body
|
73
89
|
case Benchmark::IPS.options[:format]
|
74
90
|
when :human
|
75
|
-
left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), error_percentage]
|
91
|
+
left = "%s (±%4.1f%%) i/s" % [Helpers.scale(@stats.central_tendency), @stats.error_percentage]
|
76
92
|
iters = Helpers.scale(@iterations)
|
77
93
|
|
78
94
|
if @show_total_time
|
@@ -81,7 +97,7 @@ module Benchmark
|
|
81
97
|
left.ljust(20) + (" - %s" % iters)
|
82
98
|
end
|
83
99
|
else
|
84
|
-
left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, error_percentage]
|
100
|
+
left = "%10.1f (±%.1f%%) i/s" % [@stats.central_tendency, @stats.error_percentage]
|
85
101
|
|
86
102
|
if @show_total_time
|
87
103
|
left.ljust(20) + (" - %10d in %10.6fs" % [@iterations, runtime])
|
@@ -3,27 +3,30 @@ module Benchmark
|
|
3
3
|
module Stats
|
4
4
|
|
5
5
|
class Bootstrap
|
6
|
-
|
7
|
-
attr_reader :data
|
6
|
+
include StatsMetric
|
7
|
+
attr_reader :data, :error, :samples
|
8
8
|
|
9
9
|
def initialize(samples, confidence)
|
10
10
|
dependencies
|
11
11
|
@iterations = 10_000
|
12
12
|
@confidence = (confidence / 100.0).to_s
|
13
|
+
@samples = samples
|
13
14
|
@data = Kalibera::Data.new({[0] => samples}, [1, samples.size])
|
14
15
|
interval = @data.bootstrap_confidence_interval(@iterations, @confidence)
|
15
16
|
@median = interval.median
|
16
17
|
@error = interval.error
|
17
18
|
end
|
18
19
|
|
20
|
+
# Average stat value
|
21
|
+
# @return [Float] central_tendency
|
19
22
|
def central_tendency
|
20
23
|
@median
|
21
24
|
end
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
26
|
+
# Determines how much slower this stat is than the baseline stat
|
27
|
+
# if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
|
28
|
+
# @param baseline [SD|Bootstrap] faster baseline
|
29
|
+
# @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
|
27
30
|
def slowdown(baseline)
|
28
31
|
low, slowdown, high = baseline.data.bootstrap_quotient(@data, @iterations, @confidence)
|
29
32
|
error = Timing.mean([slowdown - low, high - slowdown])
|
@@ -1,33 +1,41 @@
|
|
1
1
|
module Benchmark
|
2
2
|
module IPS
|
3
3
|
module Stats
|
4
|
-
|
4
|
+
|
5
5
|
class SD
|
6
|
-
|
6
|
+
include StatsMetric
|
7
|
+
attr_reader :error, :samples
|
8
|
+
|
7
9
|
def initialize(samples)
|
10
|
+
@samples = samples
|
8
11
|
@mean = Timing.mean(samples)
|
9
12
|
@error = Timing.stddev(samples, @mean).round
|
10
13
|
end
|
11
|
-
|
14
|
+
|
15
|
+
# Average stat value
|
16
|
+
# @return [Float] central_tendency
|
12
17
|
def central_tendency
|
13
18
|
@mean
|
14
19
|
end
|
15
|
-
|
16
|
-
def error
|
17
|
-
@error
|
18
|
-
end
|
19
20
|
|
21
|
+
# Determines how much slower this stat is than the baseline stat
|
22
|
+
# if this average is lower than the faster baseline, higher average is better (e.g. ips) (calculate accordingly)
|
23
|
+
# @param baseline [SD|Bootstrap] faster baseline
|
24
|
+
# @returns [Array<Float, nil>] the slowdown and the error (not calculated for standard deviation)
|
20
25
|
def slowdown(baseline)
|
21
|
-
|
22
|
-
|
26
|
+
if baseline.central_tendency > central_tendency
|
27
|
+
[baseline.central_tendency.to_f / central_tendency, 0]
|
28
|
+
else
|
29
|
+
[central_tendency.to_f / baseline.central_tendency, 0]
|
30
|
+
end
|
23
31
|
end
|
24
32
|
|
25
33
|
def footer
|
26
34
|
nil
|
27
35
|
end
|
28
|
-
|
36
|
+
|
29
37
|
end
|
30
|
-
|
38
|
+
|
31
39
|
end
|
32
40
|
end
|
33
41
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Benchmark
|
2
|
+
module IPS
|
3
|
+
module Stats
|
4
|
+
module StatsMetric
|
5
|
+
# Return entry's standard deviation of iteration per second in percentage.
|
6
|
+
# @return [Float] +@ips_sd+ in percentage.
|
7
|
+
def error_percentage
|
8
|
+
100.0 * (error.to_f / central_tendency)
|
9
|
+
end
|
10
|
+
|
11
|
+
def overlaps?(baseline)
|
12
|
+
baseline_low = baseline.central_tendency - baseline.error
|
13
|
+
baseline_high = baseline.central_tendency + baseline.error
|
14
|
+
my_high = central_tendency + error
|
15
|
+
my_low = central_tendency - error
|
16
|
+
my_high > baseline_low && my_low < baseline_high
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/test/test_benchmark_ips.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require "minitest/autorun"
|
2
2
|
require "benchmark/ips"
|
3
3
|
require "stringio"
|
4
|
+
require "tmpdir"
|
4
5
|
|
5
6
|
class TestBenchmarkIPS < Minitest::Test
|
6
7
|
def setup
|
@@ -20,6 +21,19 @@ class TestBenchmarkIPS < Minitest::Test
|
|
20
21
|
assert $stdout.string.size > 0
|
21
22
|
end
|
22
23
|
|
24
|
+
def test_warmup0
|
25
|
+
$stdout = @old_stdout
|
26
|
+
|
27
|
+
out, err = capture_io do
|
28
|
+
Benchmark.ips(:time => 1, :warmup => 0, :quiet => false) do |x|
|
29
|
+
x.report("sleep 0.25") { sleep(0.25) }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
refute_match(/Warming up -+/, out)
|
34
|
+
assert_empty err
|
35
|
+
end
|
36
|
+
|
23
37
|
def test_output
|
24
38
|
Benchmark.ips(1) do |x|
|
25
39
|
x.report("operation") { 100 * 100 }
|
@@ -55,11 +69,11 @@ class TestBenchmarkIPS < Minitest::Test
|
|
55
69
|
|
56
70
|
assert_equal "sleep 0.25", rep1.label
|
57
71
|
assert_equal 4, rep1.iterations
|
58
|
-
assert_in_delta 4.0, rep1.
|
72
|
+
assert_in_delta 4.0, rep1.ips, 0.2
|
59
73
|
|
60
74
|
assert_equal "sleep 0.05", rep2.label
|
61
75
|
assert_in_delta 20.0, rep2.iterations.to_f, 1.0
|
62
|
-
assert_in_delta 20.0, rep2.
|
76
|
+
assert_in_delta 20.0, rep2.ips, 2.0
|
63
77
|
end
|
64
78
|
|
65
79
|
def test_ips_alternate_config
|
@@ -73,7 +87,7 @@ class TestBenchmarkIPS < Minitest::Test
|
|
73
87
|
|
74
88
|
assert_equal "sleep 0.25", rep.label
|
75
89
|
assert_equal 4, rep.iterations
|
76
|
-
assert_in_delta 4.0, rep.
|
90
|
+
assert_in_delta 4.0, rep.ips, 0.4
|
77
91
|
end
|
78
92
|
|
79
93
|
def test_ips_old_config
|
@@ -85,7 +99,7 @@ class TestBenchmarkIPS < Minitest::Test
|
|
85
99
|
|
86
100
|
assert_equal "sleep 0.25", rep.label
|
87
101
|
assert_equal 4, rep.iterations
|
88
|
-
assert_in_delta 4.0, rep.
|
102
|
+
assert_in_delta 4.0, rep.ips, 0.2
|
89
103
|
end
|
90
104
|
|
91
105
|
def test_ips_config_suite
|
@@ -112,7 +126,7 @@ class TestBenchmarkIPS < Minitest::Test
|
|
112
126
|
|
113
127
|
assert_equal "sleep 0.25", rep.label
|
114
128
|
assert_equal 4*5, rep.iterations
|
115
|
-
assert_in_delta 4.0, rep.
|
129
|
+
assert_in_delta 4.0, rep.ips, 0.2
|
116
130
|
end
|
117
131
|
|
118
132
|
def test_ips_report_using_symbol
|
@@ -124,7 +138,7 @@ class TestBenchmarkIPS < Minitest::Test
|
|
124
138
|
|
125
139
|
assert_equal :sleep_a_quarter_second, rep.label
|
126
140
|
assert_equal 4*5, rep.iterations
|
127
|
-
assert_in_delta 4.0, rep.
|
141
|
+
assert_in_delta 4.0, rep.ips, 0.2
|
128
142
|
end
|
129
143
|
|
130
144
|
def test_ips_default_data
|
@@ -140,6 +154,17 @@ class TestBenchmarkIPS < Minitest::Test
|
|
140
154
|
assert all_data[0][:stddev]
|
141
155
|
end
|
142
156
|
|
157
|
+
def test_ips_empty
|
158
|
+
report = Benchmark.ips do |_x|
|
159
|
+
|
160
|
+
end
|
161
|
+
|
162
|
+
all_data = report.data
|
163
|
+
|
164
|
+
assert all_data
|
165
|
+
assert_equal [], all_data
|
166
|
+
end
|
167
|
+
|
143
168
|
def test_json_output
|
144
169
|
json_file = Tempfile.new("data.json")
|
145
170
|
|
@@ -158,4 +183,17 @@ class TestBenchmarkIPS < Minitest::Test
|
|
158
183
|
assert data[0]["ips"]
|
159
184
|
assert data[0]["stddev"]
|
160
185
|
end
|
186
|
+
|
187
|
+
def test_hold!
|
188
|
+
temp_file_name = Dir::Tmpname.create(["benchmark-ips", ".tmp"]) { }
|
189
|
+
|
190
|
+
Benchmark.ips(:time => 0.001, :warmup => 0.001) do |x|
|
191
|
+
x.report("operation") { 100 * 100 }
|
192
|
+
x.report("operation2") { 100 * 100 }
|
193
|
+
x.hold! temp_file_name
|
194
|
+
end
|
195
|
+
|
196
|
+
assert File.exist?(temp_file_name)
|
197
|
+
File.unlink(temp_file_name)
|
198
|
+
end
|
161
199
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark-ips
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.8.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Evan Phoenix
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: minitest
|
@@ -16,42 +16,48 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '5.
|
19
|
+
version: '5.14'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '5.
|
26
|
+
version: '5.14'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rdoc
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '4.0'
|
34
|
+
- - "<"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '7'
|
34
37
|
type: :development
|
35
38
|
prerelease: false
|
36
39
|
version_requirements: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
|
-
- - "
|
41
|
+
- - ">="
|
39
42
|
- !ruby/object:Gem::Version
|
40
43
|
version: '4.0'
|
44
|
+
- - "<"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '7'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: hoe
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
44
50
|
requirements:
|
45
51
|
- - "~>"
|
46
52
|
- !ruby/object:Gem::Version
|
47
|
-
version: '3.
|
53
|
+
version: '3.22'
|
48
54
|
type: :development
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
51
57
|
requirements:
|
52
58
|
- - "~>"
|
53
59
|
- !ruby/object:Gem::Version
|
54
|
-
version: '3.
|
60
|
+
version: '3.22'
|
55
61
|
description: An iterations per second enhancement to Benchmark.
|
56
62
|
email:
|
57
63
|
- evan@phx.io
|
@@ -63,7 +69,6 @@ extra_rdoc_files:
|
|
63
69
|
- README.md
|
64
70
|
files:
|
65
71
|
- ".autotest"
|
66
|
-
- Gemfile.lock
|
67
72
|
- History.txt
|
68
73
|
- Manifest.txt
|
69
74
|
- README.md
|
@@ -77,13 +82,15 @@ files:
|
|
77
82
|
- lib/benchmark/ips/share.rb
|
78
83
|
- lib/benchmark/ips/stats/bootstrap.rb
|
79
84
|
- lib/benchmark/ips/stats/sd.rb
|
85
|
+
- lib/benchmark/ips/stats/stats_metric.rb
|
80
86
|
- lib/benchmark/timing.rb
|
81
87
|
- test/test_benchmark_ips.rb
|
82
88
|
homepage: https://github.com/evanphx/benchmark-ips
|
83
89
|
licenses:
|
84
90
|
- MIT
|
85
|
-
metadata:
|
86
|
-
|
91
|
+
metadata:
|
92
|
+
homepage_uri: https://github.com/evanphx/benchmark-ips
|
93
|
+
post_install_message:
|
87
94
|
rdoc_options:
|
88
95
|
- "--main"
|
89
96
|
- README.md
|
@@ -100,9 +107,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
107
|
- !ruby/object:Gem::Version
|
101
108
|
version: '0'
|
102
109
|
requirements: []
|
103
|
-
|
104
|
-
|
105
|
-
signing_key:
|
110
|
+
rubygems_version: 3.1.4
|
111
|
+
signing_key:
|
106
112
|
specification_version: 4
|
107
113
|
summary: An iterations per second enhancement to Benchmark.
|
108
114
|
test_files: []
|
data/Gemfile.lock
DELETED