benchmark_suite 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Manifest.txt +0 -3
- data/README.txt +1 -1
- data/Rakefile +1 -0
- data/lib/benchmark/suite.rb +5 -2
- data/lib/benchmark_suite.rb +5 -1
- data/test/test_benchmark_suite.rb +14 -2
- metadata +41 -17
- data/lib/benchmark/compare.rb +0 -41
- data/lib/benchmark/ips.rb +0 -243
- data/lib/benchmark/timing.rb +0 -137
data/Manifest.txt
CHANGED
data/README.txt
CHANGED
data/Rakefile
CHANGED
data/lib/benchmark/suite.rb
CHANGED
@@ -2,6 +2,8 @@ require 'benchmark'
|
|
2
2
|
|
3
3
|
module Benchmark
|
4
4
|
class Suite
|
5
|
+
@current = nil
|
6
|
+
|
5
7
|
def self.current
|
6
8
|
@current
|
7
9
|
end
|
@@ -37,9 +39,10 @@ module Benchmark
|
|
37
39
|
@reports = {}
|
38
40
|
@order = []
|
39
41
|
@quiet = false
|
42
|
+
@verbose = false
|
40
43
|
end
|
41
44
|
|
42
|
-
attr_reader :reports
|
45
|
+
attr_reader :reports, :report
|
43
46
|
|
44
47
|
def quiet!
|
45
48
|
@quiet = true
|
@@ -82,7 +85,7 @@ module Benchmark
|
|
82
85
|
rescue Exception => e
|
83
86
|
STDOUT.puts "\nError in #{file}:"
|
84
87
|
if e.respond_to? :render
|
85
|
-
e.render
|
88
|
+
e.render
|
86
89
|
else
|
87
90
|
STDOUT.puts e.backtrace
|
88
91
|
end
|
data/lib/benchmark_suite.rb
CHANGED
@@ -2,7 +2,19 @@ require "test/unit"
|
|
2
2
|
require "benchmark_suite"
|
3
3
|
|
4
4
|
class TestBenchmarkSuite < Test::Unit::TestCase
|
5
|
-
def
|
6
|
-
|
5
|
+
def test_ips
|
6
|
+
s = Benchmark::Suite.create do |s|
|
7
|
+
s.quiet!
|
8
|
+
|
9
|
+
Benchmark.ips(1,1) do |x|
|
10
|
+
x.report("sleep") { sleep(0.25) }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
rep = s.report.first
|
15
|
+
|
16
|
+
assert_equal "sleep", rep.label
|
17
|
+
assert_equal 4, rep.iterations
|
18
|
+
assert_in_delta 4.0, rep.ips, 0.2
|
7
19
|
end
|
8
20
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: benchmark_suite
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
|
+
- 1
|
7
8
|
- 0
|
8
|
-
- 8
|
9
9
|
- 0
|
10
|
-
version: 0.
|
10
|
+
version: 1.0.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Evan Phoenix
|
@@ -15,25 +15,53 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date:
|
19
|
-
default_executable:
|
18
|
+
date: 2012-03-24 00:00:00 Z
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
22
|
-
name:
|
21
|
+
name: benchmark-ips
|
23
22
|
prerelease: false
|
24
23
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
24
|
none: false
|
26
25
|
requirements:
|
27
|
-
- -
|
26
|
+
- - ~>
|
28
27
|
- !ruby/object:Gem::Version
|
29
|
-
hash:
|
28
|
+
hash: 15
|
30
29
|
segments:
|
31
|
-
- 2
|
32
|
-
- 9
|
33
30
|
- 1
|
34
|
-
|
35
|
-
|
31
|
+
- 0
|
32
|
+
version: "1.0"
|
33
|
+
type: :runtime
|
36
34
|
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: rdoc
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 19
|
44
|
+
segments:
|
45
|
+
- 3
|
46
|
+
- 10
|
47
|
+
version: "3.10"
|
48
|
+
type: :development
|
49
|
+
version_requirements: *id002
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: hoe
|
52
|
+
prerelease: false
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ~>
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
hash: 31
|
59
|
+
segments:
|
60
|
+
- 2
|
61
|
+
- 14
|
62
|
+
version: "2.14"
|
63
|
+
type: :development
|
64
|
+
version_requirements: *id003
|
37
65
|
description: A set of enhancements to the standard library benchmark.rb
|
38
66
|
email:
|
39
67
|
- evan@fallingsnow.net
|
@@ -52,15 +80,11 @@ files:
|
|
52
80
|
- README.txt
|
53
81
|
- Rakefile
|
54
82
|
- bin/benchmark
|
55
|
-
- lib/benchmark/compare.rb
|
56
|
-
- lib/benchmark/ips.rb
|
57
83
|
- lib/benchmark/suite-run.rb
|
58
84
|
- lib/benchmark/suite.rb
|
59
|
-
- lib/benchmark/timing.rb
|
60
85
|
- lib/benchmark_suite.rb
|
61
86
|
- test/test_benchmark_suite.rb
|
62
87
|
- .gemtest
|
63
|
-
has_rdoc: true
|
64
88
|
homepage: http://github.com/evanphx/benchmark_suite
|
65
89
|
licenses: []
|
66
90
|
|
@@ -91,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
91
115
|
requirements: []
|
92
116
|
|
93
117
|
rubyforge_project: benchmark_suite
|
94
|
-
rubygems_version: 1.
|
118
|
+
rubygems_version: 1.8.18
|
95
119
|
signing_key:
|
96
120
|
specification_version: 3
|
97
121
|
summary: A set of enhancements to the standard library benchmark.rb
|
data/lib/benchmark/compare.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
module Benchmark
|
2
|
-
def compare(*reports)
|
3
|
-
return if reports.size < 2
|
4
|
-
|
5
|
-
iter = false
|
6
|
-
sorted = reports.sort do |a,b|
|
7
|
-
if a.respond_to? :ips
|
8
|
-
iter = true
|
9
|
-
b.ips <=> a.ips
|
10
|
-
else
|
11
|
-
a.runtime <=> b.runtime
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
best = sorted.shift
|
16
|
-
|
17
|
-
STDOUT.puts "\nComparison:"
|
18
|
-
|
19
|
-
if iter
|
20
|
-
STDOUT.printf "%20s: %10.1f i/s\n", best.label, best.ips
|
21
|
-
else
|
22
|
-
STDOUT.puts "#{best.rjust(20)}: #{best.runtime}s"
|
23
|
-
end
|
24
|
-
|
25
|
-
sorted.each do |report|
|
26
|
-
name = report.label
|
27
|
-
|
28
|
-
if iter
|
29
|
-
x = (best.ips.to_f / report.ips.to_f)
|
30
|
-
STDOUT.printf "%20s: %10.1f i/s - %.2fx slower\n", name, report.ips, x
|
31
|
-
else
|
32
|
-
x = "%.2f" % (report.ips.to_f / best.ips.to_f)
|
33
|
-
STDOUT.puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
STDOUT.puts
|
38
|
-
end
|
39
|
-
|
40
|
-
module_function :compare
|
41
|
-
end
|
data/lib/benchmark/ips.rb
DELETED
@@ -1,243 +0,0 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'benchmark/timing'
|
3
|
-
require 'benchmark/compare'
|
4
|
-
|
5
|
-
module Benchmark
|
6
|
-
|
7
|
-
class IPSReport
|
8
|
-
def initialize(label, us, iters, ips, ips_sd, cycles)
|
9
|
-
@label = label
|
10
|
-
@microseconds = us
|
11
|
-
@iterations = iters
|
12
|
-
@ips = ips
|
13
|
-
@ips_sd = ips_sd
|
14
|
-
@measurement_cycle = cycles
|
15
|
-
end
|
16
|
-
|
17
|
-
attr_reader :label, :microseconds, :iterations, :ips, :ips_sd, :measurement_cycle
|
18
|
-
|
19
|
-
def seconds
|
20
|
-
@microseconds.to_f / 1_000_000.0
|
21
|
-
end
|
22
|
-
|
23
|
-
def stddev_percentage
|
24
|
-
100.0 * (@ips_sd.to_f / @ips.to_f)
|
25
|
-
end
|
26
|
-
|
27
|
-
alias_method :runtime, :seconds
|
28
|
-
|
29
|
-
def body
|
30
|
-
left = "%10.1f (±%.1f%%) i/s" % [ips, stddev_percentage]
|
31
|
-
left.ljust(20) + (" - %10d in %10.6fs (cycle=%d)" %
|
32
|
-
[@iterations, runtime, @measurement_cycle])
|
33
|
-
end
|
34
|
-
|
35
|
-
def header
|
36
|
-
@label.rjust(20)
|
37
|
-
end
|
38
|
-
|
39
|
-
def to_s
|
40
|
-
"#{header} #{body}"
|
41
|
-
end
|
42
|
-
|
43
|
-
def display
|
44
|
-
STDOUT.puts to_s
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
class IPSJob
|
49
|
-
class Entry
|
50
|
-
def initialize(label, action)
|
51
|
-
@label = label
|
52
|
-
|
53
|
-
if action.kind_of? String
|
54
|
-
compile action
|
55
|
-
@action = self
|
56
|
-
@as_action = true
|
57
|
-
else
|
58
|
-
unless action.respond_to? :call
|
59
|
-
raise ArgumentError, "invalid action, must respond to #call"
|
60
|
-
end
|
61
|
-
|
62
|
-
@action = action
|
63
|
-
|
64
|
-
if action.respond_to? :arity and action.arity > 0
|
65
|
-
@call_loop = true
|
66
|
-
else
|
67
|
-
@call_loop = false
|
68
|
-
end
|
69
|
-
|
70
|
-
@as_action = false
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
attr_reader :label, :action
|
75
|
-
|
76
|
-
def as_action?
|
77
|
-
@as_action
|
78
|
-
end
|
79
|
-
|
80
|
-
def call_times(times)
|
81
|
-
return @action.call(times) if @call_loop
|
82
|
-
|
83
|
-
act = @action
|
84
|
-
|
85
|
-
i = 0
|
86
|
-
while i < times
|
87
|
-
act.call
|
88
|
-
i += 1
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
def compile(str)
|
93
|
-
m = (class << self; self; end)
|
94
|
-
code = <<-CODE
|
95
|
-
def call_times(__total);
|
96
|
-
__i = 0
|
97
|
-
while __i < __total
|
98
|
-
#{str};
|
99
|
-
__i += 1
|
100
|
-
end
|
101
|
-
end
|
102
|
-
CODE
|
103
|
-
m.class_eval code
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
def initialize
|
108
|
-
@list = []
|
109
|
-
@compare = false
|
110
|
-
end
|
111
|
-
|
112
|
-
attr_reader :compare
|
113
|
-
|
114
|
-
def compare!
|
115
|
-
@compare = true
|
116
|
-
end
|
117
|
-
|
118
|
-
#
|
119
|
-
# Registers the given label and block pair in the job list.
|
120
|
-
#
|
121
|
-
def item(label="", str=nil, &blk) # :yield:
|
122
|
-
if blk and str
|
123
|
-
raise ArgmentError, "specify a block and a str, but not both"
|
124
|
-
end
|
125
|
-
|
126
|
-
action = str || blk
|
127
|
-
raise ArgmentError, "no block or string" unless action
|
128
|
-
|
129
|
-
@list.push Entry.new(label, action)
|
130
|
-
self
|
131
|
-
end
|
132
|
-
|
133
|
-
alias_method :report, :item
|
134
|
-
|
135
|
-
# An array of 2-element arrays, consisting of label and block pairs.
|
136
|
-
attr_reader :list
|
137
|
-
end
|
138
|
-
|
139
|
-
def ips(time=5, warmup=2)
|
140
|
-
suite = nil
|
141
|
-
|
142
|
-
sync, STDOUT.sync = STDOUT.sync, true
|
143
|
-
|
144
|
-
if defined? Benchmark::Suite and Suite.current
|
145
|
-
suite = Benchmark::Suite.current
|
146
|
-
end
|
147
|
-
|
148
|
-
job = IPSJob.new
|
149
|
-
yield job
|
150
|
-
|
151
|
-
start = Timing::TimeVal.new
|
152
|
-
cur = Timing::TimeVal.new
|
153
|
-
|
154
|
-
before = Timing::TimeVal.new
|
155
|
-
after = Timing::TimeVal.new
|
156
|
-
|
157
|
-
reports = []
|
158
|
-
|
159
|
-
job.list.each do |item|
|
160
|
-
suite.warming item.label, warmup if suite
|
161
|
-
|
162
|
-
Timing.clean_env
|
163
|
-
|
164
|
-
if !suite or !suite.quiet?
|
165
|
-
if item.label.size > 20
|
166
|
-
STDOUT.print "#{item.label}\n#{' ' * 20}"
|
167
|
-
else
|
168
|
-
STDOUT.print item.label.rjust(20)
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
before.update!
|
173
|
-
start.update!
|
174
|
-
cur.update!
|
175
|
-
|
176
|
-
warmup_iter = 0
|
177
|
-
|
178
|
-
until start.elapsed?(cur, warmup)
|
179
|
-
item.call_times(1)
|
180
|
-
warmup_iter += 1
|
181
|
-
cur.update!
|
182
|
-
end
|
183
|
-
|
184
|
-
after.update!
|
185
|
-
|
186
|
-
warmup_time = before.diff(after)
|
187
|
-
|
188
|
-
# calculate the time to run approx 100ms
|
189
|
-
|
190
|
-
cycles_per_100ms = ((100_000 / warmup_time.to_f) * warmup_iter).to_i
|
191
|
-
cycles_per_100ms = 1 if cycles_per_100ms <= 0
|
192
|
-
|
193
|
-
suite.warmup_stats warmup_time, cycles_per_100ms if suite
|
194
|
-
|
195
|
-
Timing.clean_env
|
196
|
-
|
197
|
-
suite.running item.label, time if suite
|
198
|
-
|
199
|
-
iter = 0
|
200
|
-
start.update!
|
201
|
-
cur.update!
|
202
|
-
|
203
|
-
measurements = []
|
204
|
-
|
205
|
-
until start.elapsed?(cur, time)
|
206
|
-
before.update!
|
207
|
-
item.call_times cycles_per_100ms
|
208
|
-
after.update!
|
209
|
-
|
210
|
-
iter += cycles_per_100ms
|
211
|
-
cur.update!
|
212
|
-
|
213
|
-
measurements << before.diff(after)
|
214
|
-
end
|
215
|
-
|
216
|
-
measured_us = measurements.inject(0) { |a,i| a + i }
|
217
|
-
|
218
|
-
seconds = measured_us.to_f / 1_000_000.0
|
219
|
-
|
220
|
-
all_ips = measurements.map { |i| cycles_per_100ms.to_f / (i.to_f / 1_000_000) }
|
221
|
-
|
222
|
-
avg_ips = Timing.mean(all_ips)
|
223
|
-
sd_ips = Timing.stddev(all_ips).round
|
224
|
-
|
225
|
-
rep = IPSReport.new(item.label, measured_us, iter, avg_ips, sd_ips, cycles_per_100ms)
|
226
|
-
|
227
|
-
STDOUT.puts " #{rep.body}" if !suite or !suite.quiet?
|
228
|
-
|
229
|
-
suite.add_report rep, caller(1).first if suite
|
230
|
-
|
231
|
-
STDOUT.sync = sync
|
232
|
-
|
233
|
-
reports << rep
|
234
|
-
end
|
235
|
-
|
236
|
-
if job.compare
|
237
|
-
Benchmark.compare *reports
|
238
|
-
end
|
239
|
-
end
|
240
|
-
|
241
|
-
|
242
|
-
module_function :ips
|
243
|
-
end
|
data/lib/benchmark/timing.rb
DELETED
@@ -1,137 +0,0 @@
|
|
1
|
-
if !defined?(RUBY_ENGINE) or RUBY_ENGINE != "rbx"
|
2
|
-
require 'rubygems'
|
3
|
-
end
|
4
|
-
|
5
|
-
require 'ffi'
|
6
|
-
|
7
|
-
module Benchmark
|
8
|
-
module Timing
|
9
|
-
extend FFI::Library
|
10
|
-
|
11
|
-
ffi_lib FFI::Library::LIBC
|
12
|
-
|
13
|
-
attach_function :gettimeofday, [:pointer, :pointer], :int
|
14
|
-
|
15
|
-
class TimeVal < FFI::Struct
|
16
|
-
layout :tv_sec, :time_t, :tv_usec, :suseconds_t
|
17
|
-
|
18
|
-
def sec
|
19
|
-
self[:tv_sec]
|
20
|
-
end
|
21
|
-
|
22
|
-
def usec
|
23
|
-
self[:tv_usec]
|
24
|
-
end
|
25
|
-
|
26
|
-
def update!
|
27
|
-
Timing.gettimeofday to_ptr, nil
|
28
|
-
end
|
29
|
-
|
30
|
-
def to_time
|
31
|
-
Time.at sec, usec
|
32
|
-
end
|
33
|
-
|
34
|
-
def <(tv)
|
35
|
-
return true if sec < tv.sec
|
36
|
-
return true if usec < tv.usec
|
37
|
-
return false
|
38
|
-
end
|
39
|
-
|
40
|
-
def ==(tv)
|
41
|
-
if sec == tv.sec and usec == tv.usec
|
42
|
-
return true
|
43
|
-
end
|
44
|
-
return false
|
45
|
-
end
|
46
|
-
|
47
|
-
def diff(tv)
|
48
|
-
if tv.sec == sec
|
49
|
-
return tv.usec - usec
|
50
|
-
end
|
51
|
-
|
52
|
-
sec_diff = tv.sec - sec
|
53
|
-
|
54
|
-
usec_diff = tv.usec - usec
|
55
|
-
if usec_diff < 0
|
56
|
-
sec_diff -= 1
|
57
|
-
usec_diff += 1_000_000
|
58
|
-
end
|
59
|
-
|
60
|
-
(sec_diff * 1_000_000) + usec_diff
|
61
|
-
end
|
62
|
-
|
63
|
-
def >(tv)
|
64
|
-
tv < self
|
65
|
-
end
|
66
|
-
|
67
|
-
def elapsed?(tv, second)
|
68
|
-
target = sec + second
|
69
|
-
cur_tv = tv.sec
|
70
|
-
|
71
|
-
return true if target < cur_tv
|
72
|
-
return true if target == cur_tv and usec < tv.usec
|
73
|
-
|
74
|
-
return false
|
75
|
-
end
|
76
|
-
|
77
|
-
end
|
78
|
-
|
79
|
-
def self.resolution
|
80
|
-
samples = []
|
81
|
-
|
82
|
-
t1 = TimeVal.new
|
83
|
-
t2 = TimeVal.new
|
84
|
-
|
85
|
-
30.times do
|
86
|
-
t1.update!
|
87
|
-
|
88
|
-
while true
|
89
|
-
t2.update!
|
90
|
-
break if t2 != t1
|
91
|
-
end
|
92
|
-
|
93
|
-
samples << t1.diff(t2)
|
94
|
-
end
|
95
|
-
|
96
|
-
sum = samples.inject(0) { |acc, i| acc + i }
|
97
|
-
sum / 30
|
98
|
-
end
|
99
|
-
|
100
|
-
def self.mean(samples)
|
101
|
-
sum = samples.inject(0) { |acc, i| acc + i }
|
102
|
-
sum / samples.size
|
103
|
-
end
|
104
|
-
|
105
|
-
def self.variance(samples, m=nil)
|
106
|
-
m ||= mean(samples)
|
107
|
-
|
108
|
-
total = samples.inject(0) { |acc, i| acc + ((i - m) ** 2) }
|
109
|
-
|
110
|
-
total / samples.size
|
111
|
-
end
|
112
|
-
|
113
|
-
def self.stddev(samples, m=nil)
|
114
|
-
Math.sqrt variance(samples, m)
|
115
|
-
end
|
116
|
-
|
117
|
-
def self.resample_mean(samples, resample_times=100)
|
118
|
-
resamples = []
|
119
|
-
|
120
|
-
resample_times.times do
|
121
|
-
resample = samples.map { samples[rand(samples.size)] }
|
122
|
-
resamples << Timing.mean(resample)
|
123
|
-
end
|
124
|
-
|
125
|
-
resamples
|
126
|
-
end
|
127
|
-
|
128
|
-
def self.clean_env
|
129
|
-
# rbx
|
130
|
-
if GC.respond_to? :run
|
131
|
-
GC.run(true)
|
132
|
-
else
|
133
|
-
GC.start
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|