benchmark_suite 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.autotest +23 -0
- data/.gemtest +0 -0
- data/History.txt +6 -0
- data/Manifest.txt +13 -0
- data/README.txt +84 -0
- data/Rakefile +17 -0
- data/bin/benchmark +177 -0
- data/lib/benchmark/compare.rb +41 -0
- data/lib/benchmark/ips.rb +243 -0
- data/lib/benchmark/suite-run.rb +31 -0
- data/lib/benchmark/suite.rb +125 -0
- data/lib/benchmark/timing.rb +137 -0
- data/lib/benchmark_suite.rb +3 -0
- data/test/test_benchmark_suite.rb +8 -0
- metadata +99 -0
data/.autotest
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'autotest/restart'
|
4
|
+
|
5
|
+
# Autotest.add_hook :initialize do |at|
|
6
|
+
# at.extra_files << "../some/external/dependency.rb"
|
7
|
+
#
|
8
|
+
# at.libs << ":../some/external"
|
9
|
+
#
|
10
|
+
# at.add_exception 'vendor'
|
11
|
+
#
|
12
|
+
# at.add_mapping(/dependency.rb/) do |f, _|
|
13
|
+
# at.files_matching(/test_.*rb$/)
|
14
|
+
# end
|
15
|
+
#
|
16
|
+
# %w(TestA TestB).each do |klass|
|
17
|
+
# at.extra_class_map[klass] = "test/test_misc.rb"
|
18
|
+
# end
|
19
|
+
# end
|
20
|
+
|
21
|
+
# Autotest.add_hook :run_command do |at|
|
22
|
+
# system "rake build"
|
23
|
+
# end
|
data/.gemtest
ADDED
File without changes
|
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
.autotest
|
2
|
+
History.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/benchmark
|
7
|
+
lib/benchmark/compare.rb
|
8
|
+
lib/benchmark/ips.rb
|
9
|
+
lib/benchmark/suite-run.rb
|
10
|
+
lib/benchmark/suite.rb
|
11
|
+
lib/benchmark/timing.rb
|
12
|
+
lib/benchmark_suite.rb
|
13
|
+
test/test_benchmark_suite.rb
|
data/README.txt
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
= benchmark_suite
|
2
|
+
|
3
|
+
* http://github.com/evanphx/benchmark_suite
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
A set of enhancements to the standard library benchmark.rb
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* benchmark/ips - benchmarks a blocks iterations/second. For short snippits
|
12
|
+
of code, ips automatically figures out how many times to run the code
|
13
|
+
to get interesting data. No more guessing at random iteration counts!
|
14
|
+
|
15
|
+
* benchmark - a cli tool for running multiple benchmarks against
|
16
|
+
multiple rubies.
|
17
|
+
|
18
|
+
|
19
|
+
== SYNOPSIS:
|
20
|
+
|
21
|
+
require 'benchmark/ips'
|
22
|
+
|
23
|
+
Benchmark.ips do
|
24
|
+
# Typical mode, runs the block as many times as it can
|
25
|
+
x.report("addition") { 1 + 2 }
|
26
|
+
|
27
|
+
# To reduce overhead, the number of iterations is passed in
|
28
|
+
# and the block must run the code the specific number of times.
|
29
|
+
# Used for when the workload is very small and any overhead
|
30
|
+
# introduces incorrectable errors.
|
31
|
+
x.report("addition2") do |times|
|
32
|
+
i = 0
|
33
|
+
while i < times
|
34
|
+
1 + 2
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# To reduce overhead even more, grafts the code given into
|
39
|
+
# the loop that performs the iterations internally to reduce
|
40
|
+
# overhead. Typically not needed, use the |times| form instead.
|
41
|
+
x.report("addition3", "1 + 2")
|
42
|
+
end
|
43
|
+
|
44
|
+
== REQUIREMENTS:
|
45
|
+
|
46
|
+
* ruby-ffi on MRI
|
47
|
+
|
48
|
+
== INSTALL:
|
49
|
+
|
50
|
+
* gem install benchmark-suite
|
51
|
+
|
52
|
+
== DEVELOPERS:
|
53
|
+
|
54
|
+
After checking out the source, run:
|
55
|
+
|
56
|
+
$ rake newb
|
57
|
+
|
58
|
+
This task will install any missing dependencies, run the tests/specs,
|
59
|
+
and generate the RDoc.
|
60
|
+
|
61
|
+
== LICENSE:
|
62
|
+
|
63
|
+
(The MIT License)
|
64
|
+
|
65
|
+
Copyright (c) 2011 Evan Phoenix
|
66
|
+
|
67
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
68
|
+
a copy of this software and associated documentation files (the
|
69
|
+
'Software'), to deal in the Software without restriction, including
|
70
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
71
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
72
|
+
permit persons to whom the Software is furnished to do so, subject to
|
73
|
+
the following conditions:
|
74
|
+
|
75
|
+
The above copyright notice and this permission notice shall be
|
76
|
+
included in all copies or substantial portions of the Software.
|
77
|
+
|
78
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
79
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
80
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
81
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
82
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
83
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
84
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
|
6
|
+
# Hoe.plugin :compiler
|
7
|
+
# Hoe.plugin :gem_prelude_sucks
|
8
|
+
# Hoe.plugin :inline
|
9
|
+
# Hoe.plugin :inline
|
10
|
+
# Hoe.plugin :racc
|
11
|
+
# Hoe.plugin :rubyforge
|
12
|
+
|
13
|
+
Hoe.spec 'benchmark_suite' do
|
14
|
+
developer('Evan Phoenix', 'evan@fallingsnow.net')
|
15
|
+
end
|
16
|
+
|
17
|
+
# vim: syntax=ruby
|
data/bin/benchmark
ADDED
@@ -0,0 +1,177 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'tempfile'
|
5
|
+
|
6
|
+
lib_path = File.expand_path("../../lib", __FILE__)
|
7
|
+
|
8
|
+
$:.unshift lib_path
|
9
|
+
|
10
|
+
require 'benchmark/suite'
|
11
|
+
require 'benchmark/ips'
|
12
|
+
|
13
|
+
targets = []
|
14
|
+
extra = []
|
15
|
+
at_end = false
|
16
|
+
save = false
|
17
|
+
|
18
|
+
compare_targets = []
|
19
|
+
|
20
|
+
opt = OptionParser.new do |o|
|
21
|
+
o.on("-t", "--target TARGET", String,
|
22
|
+
"Use TARGET to compare against: r:ruby|r19:ruby19|x:rbx|j:jruby") do |t|
|
23
|
+
case t
|
24
|
+
when 'r', 'ruby'
|
25
|
+
targets << 'ruby'
|
26
|
+
when 'r19', 'ruby19'
|
27
|
+
targets << 'ruby19'
|
28
|
+
when 'x', 'rbx', 'rubinius'
|
29
|
+
targets << 'rbx'
|
30
|
+
when 'j', 'jruby'
|
31
|
+
targets << 'jruby'
|
32
|
+
else
|
33
|
+
# + disambiguates execution vs using a file
|
34
|
+
if t[0,1] == "+"
|
35
|
+
targets << t[1..-1]
|
36
|
+
elsif File.exists?(t)
|
37
|
+
begin
|
38
|
+
data = Marshal.load(File.read(t))
|
39
|
+
compare_targets << [t, data]
|
40
|
+
rescue TypeError
|
41
|
+
targets << t
|
42
|
+
end
|
43
|
+
else
|
44
|
+
targets << t
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
o.on("-p", "--profile", "Profile code while benchmarking (rbx only)") do
|
50
|
+
extra << "-Xprofile"
|
51
|
+
end
|
52
|
+
|
53
|
+
o.on("-e", "--end", "Report all stats after all suitse have run") do
|
54
|
+
at_end = true
|
55
|
+
end
|
56
|
+
|
57
|
+
o.on("-T OPTION", String, "Add more arguments to each target") do |t|
|
58
|
+
extra << t
|
59
|
+
end
|
60
|
+
|
61
|
+
o.on("-s", "--save PATH", String, "Save the results to a file") do |t|
|
62
|
+
save = t
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
opt.parse!
|
67
|
+
|
68
|
+
if targets.empty?
|
69
|
+
targets << "ruby"
|
70
|
+
end
|
71
|
+
|
72
|
+
if save and targets.size > 1
|
73
|
+
STDOUT.puts "Save mode only available with one target."
|
74
|
+
exit 1
|
75
|
+
end
|
76
|
+
|
77
|
+
opts = []
|
78
|
+
|
79
|
+
if at_end
|
80
|
+
opts << "--quiet"
|
81
|
+
end
|
82
|
+
|
83
|
+
results = targets.map do |t|
|
84
|
+
tf = Tempfile.new "benchmark"
|
85
|
+
tf.close
|
86
|
+
STDOUT.puts "=== #{t} ===" unless at_end
|
87
|
+
|
88
|
+
args = extra + ["-I#{lib_path}", "#{lib_path}/benchmark/suite-run.rb"]
|
89
|
+
|
90
|
+
args += opts
|
91
|
+
args << tf.path
|
92
|
+
args += ARGV
|
93
|
+
|
94
|
+
cmd, *rest = t.split(/\s+/)
|
95
|
+
args.unshift *rest
|
96
|
+
|
97
|
+
system cmd, *args
|
98
|
+
|
99
|
+
if $?.exitstatus != 0
|
100
|
+
puts "Error executing: #{cmd}"
|
101
|
+
nil
|
102
|
+
else
|
103
|
+
tf.open
|
104
|
+
[t, Marshal.load(tf.read)]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
results.compact!
|
109
|
+
|
110
|
+
if save
|
111
|
+
name, data = results.last
|
112
|
+
|
113
|
+
File.open save, "w" do |f|
|
114
|
+
f << Marshal.dump(data)
|
115
|
+
end
|
116
|
+
|
117
|
+
STDOUT.puts "[Saved results to '#{save}']"
|
118
|
+
end
|
119
|
+
|
120
|
+
if at_end
|
121
|
+
results.each do |name, suite|
|
122
|
+
STDOUT.puts "=== #{name} ==="
|
123
|
+
suite.display
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
results += compare_targets
|
128
|
+
|
129
|
+
if results.size > 1
|
130
|
+
compared = Hash.new { |h,k| h[k] = [] }
|
131
|
+
|
132
|
+
results.each do |target, suite|
|
133
|
+
suite.reports.each do |name, reports|
|
134
|
+
reports.each do |rep|
|
135
|
+
compared["#{name}:#{rep.label}"] << [target, rep]
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
STDOUT.puts
|
141
|
+
|
142
|
+
compared.each do |name, reports|
|
143
|
+
if reports.size > 1
|
144
|
+
STDOUT.puts "Comparing #{name}:"
|
145
|
+
|
146
|
+
iter = false
|
147
|
+
sorted = reports.sort do |a,b|
|
148
|
+
if a[1].respond_to? :ips
|
149
|
+
iter = true
|
150
|
+
b[1].ips <=> a[1].ips
|
151
|
+
else
|
152
|
+
a[1].runtime <=> b[1].runtime
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
best_name, best_report = sorted.shift
|
157
|
+
|
158
|
+
|
159
|
+
if iter
|
160
|
+
STDOUT.printf "%20s: %10d i/s\n", best_name, best_report.ips
|
161
|
+
else
|
162
|
+
STDOUT.puts "#{best_name.rjust(20)}: #{best_report.runtime}s"
|
163
|
+
end
|
164
|
+
|
165
|
+
sorted.each do |entry|
|
166
|
+
name, report = entry
|
167
|
+
if iter
|
168
|
+
x = (best_report.ips.to_f / report.ips.to_f)
|
169
|
+
STDOUT.printf "%20s: %10d i/s - %.2fx slower\n", name, report.ips, x
|
170
|
+
else
|
171
|
+
x = "%.2f" % (report.ips.to_f / best_report.ips.to_f)
|
172
|
+
STDOUT.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Benchmark
|
2
|
+
def compare(*reports)
|
3
|
+
return if reports.size < 2
|
4
|
+
|
5
|
+
iter = false
|
6
|
+
sorted = reports.sort do |a,b|
|
7
|
+
if a.respond_to? :ips
|
8
|
+
iter = true
|
9
|
+
b.ips <=> a.ips
|
10
|
+
else
|
11
|
+
a.runtime <=> b.runtime
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
best = sorted.shift
|
16
|
+
|
17
|
+
STDOUT.puts "\nComparison:"
|
18
|
+
|
19
|
+
if iter
|
20
|
+
STDOUT.printf "%20s: %10.1f i/s\n", best.label, best.ips
|
21
|
+
else
|
22
|
+
STDOUT.puts "#{best.rjust(20)}: #{best.runtime}s"
|
23
|
+
end
|
24
|
+
|
25
|
+
sorted.each do |report|
|
26
|
+
name = report.label
|
27
|
+
|
28
|
+
if iter
|
29
|
+
x = (best.ips.to_f / report.ips.to_f)
|
30
|
+
STDOUT.printf "%20s: %10.1f i/s - %.2fx slower\n", name, report.ips, x
|
31
|
+
else
|
32
|
+
x = "%.2f" % (report.ips.to_f / best.ips.to_f)
|
33
|
+
STDOUT.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
STDOUT.puts
|
38
|
+
end
|
39
|
+
|
40
|
+
module_function :compare
|
41
|
+
end
|
@@ -0,0 +1,243 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'benchmark/timing'
|
3
|
+
require 'benchmark/compare'
|
4
|
+
|
5
|
+
module Benchmark
|
6
|
+
|
7
|
+
class IPSReport
|
8
|
+
def initialize(label, us, iters, ips, ips_sd, cycles)
|
9
|
+
@label = label
|
10
|
+
@microseconds = us
|
11
|
+
@iterations = iters
|
12
|
+
@ips = ips
|
13
|
+
@ips_sd = ips_sd
|
14
|
+
@measurement_cycle = cycles
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :label, :microseconds, :iterations, :ips, :ips_sd, :measurement_cycle
|
18
|
+
|
19
|
+
def seconds
|
20
|
+
@microseconds.to_f / 1_000_000.0
|
21
|
+
end
|
22
|
+
|
23
|
+
def stddev_percentage
|
24
|
+
100.0 * (@ips_sd.to_f / @ips.to_f)
|
25
|
+
end
|
26
|
+
|
27
|
+
alias_method :runtime, :seconds
|
28
|
+
|
29
|
+
def body
|
30
|
+
left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
|
31
|
+
left.ljust(20) + (" - %10d in %10.6fs (cycle=%d)" %
|
32
|
+
[@iterations, runtime, @measurement_cycle])
|
33
|
+
end
|
34
|
+
|
35
|
+
def header
|
36
|
+
@label.rjust(20)
|
37
|
+
end
|
38
|
+
|
39
|
+
def to_s
|
40
|
+
"#{header} #{body}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def display
|
44
|
+
STDOUT.puts to_s
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
class IPSJob
|
49
|
+
class Entry
|
50
|
+
def initialize(label, action)
|
51
|
+
@label = label
|
52
|
+
|
53
|
+
if action.kind_of? String
|
54
|
+
compile action
|
55
|
+
@action = self
|
56
|
+
@as_action = true
|
57
|
+
else
|
58
|
+
unless action.respond_to? :call
|
59
|
+
raise ArgumentError, "invalid action, must respond to #call"
|
60
|
+
end
|
61
|
+
|
62
|
+
@action = action
|
63
|
+
|
64
|
+
if action.respond_to? :arity and action.arity > 0
|
65
|
+
@call_loop = true
|
66
|
+
else
|
67
|
+
@call_loop = false
|
68
|
+
end
|
69
|
+
|
70
|
+
@as_action = false
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
attr_reader :label, :action
|
75
|
+
|
76
|
+
def as_action?
|
77
|
+
@as_action
|
78
|
+
end
|
79
|
+
|
80
|
+
def call_times(times)
|
81
|
+
return @action.call(times) if @call_loop
|
82
|
+
|
83
|
+
act = @action
|
84
|
+
|
85
|
+
i = 0
|
86
|
+
while i < times
|
87
|
+
act.call
|
88
|
+
i += 1
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def compile(str)
|
93
|
+
m = (class << self; self; end)
|
94
|
+
code = <<-CODE
|
95
|
+
def call_times(__total);
|
96
|
+
__i = 0
|
97
|
+
while __i < __total
|
98
|
+
#{str};
|
99
|
+
__i += 1
|
100
|
+
end
|
101
|
+
end
|
102
|
+
CODE
|
103
|
+
m.class_eval code
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def initialize
|
108
|
+
@list = []
|
109
|
+
@compare = false
|
110
|
+
end
|
111
|
+
|
112
|
+
attr_reader :compare
|
113
|
+
|
114
|
+
def compare!
|
115
|
+
@compare = true
|
116
|
+
end
|
117
|
+
|
118
|
+
#
|
119
|
+
# Registers the given label and block pair in the job list.
|
120
|
+
#
|
121
|
+
def item(label="", str=nil, &blk) # :yield:
|
122
|
+
if blk and str
|
123
|
+
raise ArgmentError, "specify a block and a str, but not both"
|
124
|
+
end
|
125
|
+
|
126
|
+
action = str || blk
|
127
|
+
raise ArgmentError, "no block or string" unless action
|
128
|
+
|
129
|
+
@list.push Entry.new(label, action)
|
130
|
+
self
|
131
|
+
end
|
132
|
+
|
133
|
+
alias_method :report, :item
|
134
|
+
|
135
|
+
# An array of 2-element arrays, consisting of label and block pairs.
|
136
|
+
attr_reader :list
|
137
|
+
end
|
138
|
+
|
139
|
+
def ips(time=5, warmup=2)
|
140
|
+
suite = nil
|
141
|
+
|
142
|
+
sync, STDOUT.sync = STDOUT.sync, true
|
143
|
+
|
144
|
+
if defined? Benchmark::Suite and Suite.current
|
145
|
+
suite = Benchmark::Suite.current
|
146
|
+
end
|
147
|
+
|
148
|
+
job = IPSJob.new
|
149
|
+
yield job
|
150
|
+
|
151
|
+
start = Timing::TimeVal.new
|
152
|
+
cur = Timing::TimeVal.new
|
153
|
+
|
154
|
+
before = Timing::TimeVal.new
|
155
|
+
after = Timing::TimeVal.new
|
156
|
+
|
157
|
+
reports = []
|
158
|
+
|
159
|
+
job.list.each do |item|
|
160
|
+
suite.warming item.label, warmup if suite
|
161
|
+
|
162
|
+
Timing.clean_env
|
163
|
+
|
164
|
+
if !suite or !suite.quiet?
|
165
|
+
if item.label.size > 20
|
166
|
+
STDOUT.print "#{item.label}\n#{' ' * 20}"
|
167
|
+
else
|
168
|
+
STDOUT.print item.label.rjust(20)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
before.update!
|
173
|
+
start.update!
|
174
|
+
cur.update!
|
175
|
+
|
176
|
+
warmup_iter = 0
|
177
|
+
|
178
|
+
until start.elapsed?(cur, warmup)
|
179
|
+
item.call_times(1)
|
180
|
+
warmup_iter += 1
|
181
|
+
cur.update!
|
182
|
+
end
|
183
|
+
|
184
|
+
after.update!
|
185
|
+
|
186
|
+
warmup_time = before.diff(after)
|
187
|
+
|
188
|
+
# calculate the time to run approx 100ms
|
189
|
+
|
190
|
+
cycles_per_100ms = ((100_000 / warmup_time.to_f) * warmup_iter).to_i
|
191
|
+
cycles_per_100ms = 1 if cycles_per_100ms <= 0
|
192
|
+
|
193
|
+
suite.warmup_stats warmup_time, cycles_per_100ms if suite
|
194
|
+
|
195
|
+
Timing.clean_env
|
196
|
+
|
197
|
+
suite.running item.label, time if suite
|
198
|
+
|
199
|
+
iter = 0
|
200
|
+
start.update!
|
201
|
+
cur.update!
|
202
|
+
|
203
|
+
measurements = []
|
204
|
+
|
205
|
+
until start.elapsed?(cur, time)
|
206
|
+
before.update!
|
207
|
+
item.call_times cycles_per_100ms
|
208
|
+
after.update!
|
209
|
+
|
210
|
+
iter += cycles_per_100ms
|
211
|
+
cur.update!
|
212
|
+
|
213
|
+
measurements << before.diff(after)
|
214
|
+
end
|
215
|
+
|
216
|
+
measured_us = measurements.inject(0) { |a,i| a + i }
|
217
|
+
|
218
|
+
seconds = measured_us.to_f / 1_000_000.0
|
219
|
+
|
220
|
+
all_ips = measurements.map { |i| cycles_per_100ms.to_f / (i.to_f / 1_000_000) }
|
221
|
+
|
222
|
+
avg_ips = Timing.mean(all_ips)
|
223
|
+
sd_ips = Timing.stddev(all_ips).round
|
224
|
+
|
225
|
+
rep = IPSReport.new(item.label, measured_us, iter, avg_ips, sd_ips, cycles_per_100ms)
|
226
|
+
|
227
|
+
STDOUT.puts " #{rep.body}" if !suite or !suite.quiet?
|
228
|
+
|
229
|
+
suite.add_report rep, caller(1).first if suite
|
230
|
+
|
231
|
+
STDOUT.sync = sync
|
232
|
+
|
233
|
+
reports << rep
|
234
|
+
end
|
235
|
+
|
236
|
+
if job.compare
|
237
|
+
Benchmark.compare *reports
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
|
242
|
+
module_function :ips
|
243
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'benchmark/suite'
|
2
|
+
require 'optparse'
|
3
|
+
|
4
|
+
quiet = false
|
5
|
+
|
6
|
+
opts = OptionParser.new do |o|
|
7
|
+
o.on "-q", "--quiet" do
|
8
|
+
quiet = true
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
opts.parse!
|
13
|
+
|
14
|
+
results = ARGV.shift
|
15
|
+
|
16
|
+
suite = Benchmark::Suite.create do |s|
|
17
|
+
s.quiet! if quiet
|
18
|
+
|
19
|
+
ARGV.each do |f|
|
20
|
+
if File.directory?(f)
|
21
|
+
more = Dir["#{f}/**/{bench,bm}_*.rb"]
|
22
|
+
more.each do |x|
|
23
|
+
s.run x
|
24
|
+
end
|
25
|
+
else
|
26
|
+
s.run f
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
File.open(results, "w") { |f| f << Marshal.dump(suite) }
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'benchmark'
|
2
|
+
|
3
|
+
module Benchmark
|
4
|
+
class Suite
|
5
|
+
def self.current
|
6
|
+
@current
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.create
|
10
|
+
if block_given?
|
11
|
+
old = @current
|
12
|
+
|
13
|
+
begin
|
14
|
+
s = new
|
15
|
+
@current = s
|
16
|
+
yield s
|
17
|
+
return s
|
18
|
+
ensure
|
19
|
+
@current = old
|
20
|
+
end
|
21
|
+
else
|
22
|
+
@current = new
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class SimpleReport
|
27
|
+
def initialize
|
28
|
+
@start = nil
|
29
|
+
@end = nil
|
30
|
+
end
|
31
|
+
|
32
|
+
attr_reader :start, :end
|
33
|
+
end
|
34
|
+
|
35
|
+
def initialize
|
36
|
+
@report = nil
|
37
|
+
@reports = {}
|
38
|
+
@order = []
|
39
|
+
@quiet = false
|
40
|
+
end
|
41
|
+
|
42
|
+
attr_reader :reports
|
43
|
+
|
44
|
+
def quiet!
|
45
|
+
@quiet = true
|
46
|
+
end
|
47
|
+
|
48
|
+
def quiet?
|
49
|
+
@quiet
|
50
|
+
end
|
51
|
+
|
52
|
+
def warming(label, sec)
|
53
|
+
return unless @verbose
|
54
|
+
STDOUT.print "#{label.rjust(20)} warmup: #{sec}s"
|
55
|
+
end
|
56
|
+
|
57
|
+
def warmup_stats(time, cycles)
|
58
|
+
return unless @verbose
|
59
|
+
STDOUT.print " time=#{time}us, cycles=#{cycles}."
|
60
|
+
end
|
61
|
+
|
62
|
+
def running(label, sec)
|
63
|
+
return unless @verbose
|
64
|
+
STDOUT.puts " running: #{sec}s..."
|
65
|
+
end
|
66
|
+
|
67
|
+
def add_report(rep, location)
|
68
|
+
if @report
|
69
|
+
@report << rep
|
70
|
+
else
|
71
|
+
@report = [rep]
|
72
|
+
end
|
73
|
+
|
74
|
+
@report_location = location
|
75
|
+
end
|
76
|
+
|
77
|
+
def run(file)
|
78
|
+
start = Time.now
|
79
|
+
|
80
|
+
begin
|
81
|
+
load file
|
82
|
+
rescue Exception => e
|
83
|
+
STDOUT.puts "\nError in #{file}:"
|
84
|
+
if e.respond_to? :render
|
85
|
+
e.render(STDERR)
|
86
|
+
else
|
87
|
+
STDOUT.puts e.backtrace
|
88
|
+
end
|
89
|
+
return
|
90
|
+
end
|
91
|
+
|
92
|
+
fin = Time.now
|
93
|
+
|
94
|
+
if @report
|
95
|
+
@reports[file] = @report
|
96
|
+
@report = nil
|
97
|
+
else
|
98
|
+
@reports[file] = SimpleReport.new(start, fin)
|
99
|
+
end
|
100
|
+
|
101
|
+
@order << file
|
102
|
+
end
|
103
|
+
|
104
|
+
def display
|
105
|
+
if @report
|
106
|
+
file = @report_location ? @report_location.split(":").first : "<unknown.rb>"
|
107
|
+
@order << file
|
108
|
+
@reports[file] = [@report]
|
109
|
+
end
|
110
|
+
|
111
|
+
@order.each do |file|
|
112
|
+
STDOUT.puts "#{file}:"
|
113
|
+
reports = @reports[file]
|
114
|
+
|
115
|
+
if reports.empty?
|
116
|
+
STDOUT.puts " NO REPORTS FOUND"
|
117
|
+
else
|
118
|
+
reports.each do |rep|
|
119
|
+
STDOUT.puts " #{rep}"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
if !defined?(RUBY_ENGINE) or RUBY_ENGINE != "rbx"
|
2
|
+
require 'rubygems'
|
3
|
+
end
|
4
|
+
|
5
|
+
require 'ffi'
|
6
|
+
|
7
|
+
module Benchmark
|
8
|
+
module Timing
|
9
|
+
extend FFI::Library
|
10
|
+
|
11
|
+
ffi_lib FFI::Library::LIBC
|
12
|
+
|
13
|
+
attach_function :gettimeofday, [:pointer, :pointer], :int
|
14
|
+
|
15
|
+
class TimeVal < FFI::Struct
|
16
|
+
layout :tv_sec, :time_t, :tv_usec, :suseconds_t
|
17
|
+
|
18
|
+
def sec
|
19
|
+
self[:tv_sec]
|
20
|
+
end
|
21
|
+
|
22
|
+
def usec
|
23
|
+
self[:tv_usec]
|
24
|
+
end
|
25
|
+
|
26
|
+
def update!
|
27
|
+
Timing.gettimeofday to_ptr, nil
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_time
|
31
|
+
Time.at sec, usec
|
32
|
+
end
|
33
|
+
|
34
|
+
def <(tv)
|
35
|
+
return true if sec < tv.sec
|
36
|
+
return true if usec < tv.usec
|
37
|
+
return false
|
38
|
+
end
|
39
|
+
|
40
|
+
def ==(tv)
|
41
|
+
if sec == tv.sec and usec == tv.usec
|
42
|
+
return true
|
43
|
+
end
|
44
|
+
return false
|
45
|
+
end
|
46
|
+
|
47
|
+
def diff(tv)
|
48
|
+
if tv.sec == sec
|
49
|
+
return tv.usec - usec
|
50
|
+
end
|
51
|
+
|
52
|
+
sec_diff = tv.sec - sec
|
53
|
+
|
54
|
+
usec_diff = tv.usec - usec
|
55
|
+
if usec_diff < 0
|
56
|
+
sec_diff -= 1
|
57
|
+
usec_diff += 1_000_000
|
58
|
+
end
|
59
|
+
|
60
|
+
(sec_diff * 1_000_000) + usec_diff
|
61
|
+
end
|
62
|
+
|
63
|
+
def >(tv)
|
64
|
+
tv < self
|
65
|
+
end
|
66
|
+
|
67
|
+
def elapsed?(tv, second)
|
68
|
+
target = sec + second
|
69
|
+
cur_tv = tv.sec
|
70
|
+
|
71
|
+
return true if target < cur_tv
|
72
|
+
return true if target == cur_tv and usec < tv.usec
|
73
|
+
|
74
|
+
return false
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.resolution
|
80
|
+
samples = []
|
81
|
+
|
82
|
+
t1 = TimeVal.new
|
83
|
+
t2 = TimeVal.new
|
84
|
+
|
85
|
+
30.times do
|
86
|
+
t1.update!
|
87
|
+
|
88
|
+
while true
|
89
|
+
t2.update!
|
90
|
+
break if t2 != t1
|
91
|
+
end
|
92
|
+
|
93
|
+
samples << t1.diff(t2)
|
94
|
+
end
|
95
|
+
|
96
|
+
sum = samples.inject(0) { |acc, i| acc + i }
|
97
|
+
sum / 30
|
98
|
+
end
|
99
|
+
|
100
|
+
def self.mean(samples)
|
101
|
+
sum = samples.inject(0) { |acc, i| acc + i }
|
102
|
+
sum / samples.size
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.variance(samples, m=nil)
|
106
|
+
m ||= mean(samples)
|
107
|
+
|
108
|
+
total = samples.inject(0) { |acc, i| acc + ((i - m) ** 2) }
|
109
|
+
|
110
|
+
total / samples.size
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.stddev(samples, m=nil)
|
114
|
+
Math.sqrt variance(samples, m)
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.resample_mean(samples, resample_times=100)
|
118
|
+
resamples = []
|
119
|
+
|
120
|
+
resample_times.times do
|
121
|
+
resample = samples.map { samples[rand(samples.size)] }
|
122
|
+
resamples << Timing.mean(resample)
|
123
|
+
end
|
124
|
+
|
125
|
+
resamples
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.clean_env
|
129
|
+
# rbx
|
130
|
+
if GC.respond_to? :run
|
131
|
+
GC.run(true)
|
132
|
+
else
|
133
|
+
GC.start
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
metadata
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: benchmark_suite
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 63
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 8
|
9
|
+
- 0
|
10
|
+
version: 0.8.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Evan Phoenix
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-04-05 00:00:00 -07:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: hoe
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 41
|
30
|
+
segments:
|
31
|
+
- 2
|
32
|
+
- 9
|
33
|
+
- 1
|
34
|
+
version: 2.9.1
|
35
|
+
type: :development
|
36
|
+
version_requirements: *id001
|
37
|
+
description: A set of enhancements to the standard library benchmark.rb
|
38
|
+
email:
|
39
|
+
- evan@fallingsnow.net
|
40
|
+
executables:
|
41
|
+
- benchmark
|
42
|
+
extensions: []
|
43
|
+
|
44
|
+
extra_rdoc_files:
|
45
|
+
- History.txt
|
46
|
+
- Manifest.txt
|
47
|
+
- README.txt
|
48
|
+
files:
|
49
|
+
- .autotest
|
50
|
+
- History.txt
|
51
|
+
- Manifest.txt
|
52
|
+
- README.txt
|
53
|
+
- Rakefile
|
54
|
+
- bin/benchmark
|
55
|
+
- lib/benchmark/compare.rb
|
56
|
+
- lib/benchmark/ips.rb
|
57
|
+
- lib/benchmark/suite-run.rb
|
58
|
+
- lib/benchmark/suite.rb
|
59
|
+
- lib/benchmark/timing.rb
|
60
|
+
- lib/benchmark_suite.rb
|
61
|
+
- test/test_benchmark_suite.rb
|
62
|
+
- .gemtest
|
63
|
+
has_rdoc: true
|
64
|
+
homepage: http://github.com/evanphx/benchmark_suite
|
65
|
+
licenses: []
|
66
|
+
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options:
|
69
|
+
- --main
|
70
|
+
- README.txt
|
71
|
+
require_paths:
|
72
|
+
- lib
|
73
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
hash: 3
|
79
|
+
segments:
|
80
|
+
- 0
|
81
|
+
version: "0"
|
82
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ">="
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
hash: 3
|
88
|
+
segments:
|
89
|
+
- 0
|
90
|
+
version: "0"
|
91
|
+
requirements: []
|
92
|
+
|
93
|
+
rubyforge_project: benchmark_suite
|
94
|
+
rubygems_version: 1.6.2
|
95
|
+
signing_key:
|
96
|
+
specification_version: 3
|
97
|
+
summary: A set of enhancements to the standard library benchmark.rb
|
98
|
+
test_files:
|
99
|
+
- test/test_benchmark_suite.rb
|