benchmark-bigo 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +119 -15
- data/lib/benchmark/bigo.rb +4 -8
- data/lib/benchmark/bigo/chart.rb +148 -0
- data/lib/benchmark/bigo/job.rb +102 -75
- data/lib/benchmark/bigo/report.rb +9 -93
- data/lib/benchmark/bigo/templates/chart.erb +1 -0
- data/lib/benchmark/bigo/termplot.rb +70 -0
- data/lib/benchmark/bigo/version.rb +2 -2
- data/test/benchmark/test_bigo.rb +174 -82
- data/test/benchmark/test_chart.rb +89 -0
- data/test/benchmark/test_termplot.rb +52 -0
- metadata +8 -2
@@ -9,15 +9,6 @@ module Benchmark
|
|
9
9
|
def initialize
|
10
10
|
@per_iterations = 0
|
11
11
|
@entries = {}
|
12
|
-
@logscale = false
|
13
|
-
end
|
14
|
-
|
15
|
-
def logscale?
|
16
|
-
@logscale
|
17
|
-
end
|
18
|
-
|
19
|
-
def logscale!
|
20
|
-
@logscale = true
|
21
12
|
end
|
22
13
|
|
23
14
|
def add_entry label, microseconds, iters, ips, ips_sd, measurement_cycle
|
@@ -28,103 +19,28 @@ module Benchmark
|
|
28
19
|
@entries[group_label].last
|
29
20
|
end
|
30
21
|
|
31
|
-
|
32
|
-
|
33
|
-
end
|
34
|
-
|
35
|
-
def chart_hash group_label
|
22
|
+
# retrieve benchmark data for a particular label
|
23
|
+
def data_for group_label
|
36
24
|
@entries[group_label].collect do |report|
|
37
25
|
size = report.label.split(' ').last.to_i
|
38
|
-
|
26
|
+
microseconds_per_iters = 1000000.0 / report.ips.to_f
|
39
27
|
|
40
28
|
{label: size,
|
41
|
-
|
29
|
+
microseconds_per_iters: microseconds_per_iters,
|
42
30
|
ips: report.ips
|
43
31
|
}
|
44
32
|
end
|
45
33
|
end
|
46
34
|
|
47
|
-
|
48
|
-
|
49
|
-
Hash[chart_hash.collect{|h| [h[:label], h[:seconds_per_scaled_iters]]}]
|
50
|
-
end
|
51
|
-
|
52
|
-
def chart_data
|
35
|
+
# retrieve a summary of data for the benchmark report
|
36
|
+
def data
|
53
37
|
@entries.keys.map do |k|
|
54
|
-
|
55
|
-
{
|
56
|
-
|
57
|
-
end
|
58
|
-
|
59
|
-
def chart_opts chart_data
|
60
|
-
|
61
|
-
axis_type = logscale? ? 'logarithmic' : 'linear'
|
62
|
-
|
63
|
-
if chart_data.is_a? Array
|
64
|
-
min = chart_data.collect{|d| d[:data].values.min}.min
|
65
|
-
max = chart_data.collect{|d| d[:data].values.max}.max
|
66
|
-
|
67
|
-
elsif chart_data.is_a? Hash
|
68
|
-
min = chart_data[:data].values.min
|
69
|
-
max = chart_data[:data].values.max
|
38
|
+
key_data = data_for(k)
|
39
|
+
data = Hash[key_data.collect{|h| [h[:label], h[:microseconds_per_iters]]}]
|
40
|
+
{name: k, data: data}
|
70
41
|
end
|
71
|
-
|
72
|
-
orange = "#f0662d"
|
73
|
-
purple = "#8062a6"
|
74
|
-
light_green = "#7bc545"
|
75
|
-
med_blue = "#0883b2"
|
76
|
-
yellow = "#ffaa00"
|
77
|
-
|
78
|
-
{
|
79
|
-
discrete: true,
|
80
|
-
width: "800px",
|
81
|
-
height: "500px",
|
82
|
-
min: (min * 0.8).floor,
|
83
|
-
max: (max * 1.2).ceil,
|
84
|
-
library: {
|
85
|
-
colors: [orange, purple, light_green, med_blue, yellow],
|
86
|
-
xAxis: {type: axis_type, title: {text: "Size"}},
|
87
|
-
yAxis: {type: axis_type, title: {text: "Seconds per #{scaled_iterations.to_i} Iterations"}}
|
88
|
-
}
|
89
|
-
}
|
90
42
|
end
|
91
43
|
|
92
|
-
def comparison_chart_data chart_data, sizes
|
93
|
-
sample_size = sizes.first
|
94
|
-
|
95
|
-
# can't take log of 1,
|
96
|
-
# so it can't be used as the sample
|
97
|
-
if sample_size == 1
|
98
|
-
sample_size = sizes[1]
|
99
|
-
end
|
100
|
-
|
101
|
-
sample = chart_data[:data][sample_size]
|
102
|
-
|
103
|
-
logn_sample = sample/Math.log10(sample_size)
|
104
|
-
n_sample = sample/sample_size
|
105
|
-
nlogn_sample = sample/(sample_size * Math.log10(sample_size))
|
106
|
-
n2_sample = sample/(sample_size * sample_size)
|
107
|
-
|
108
|
-
logn_data = {}
|
109
|
-
n_data = {}
|
110
|
-
nlogn_data = {}
|
111
|
-
n2_data = {}
|
112
|
-
|
113
|
-
sizes.each do |n|
|
114
|
-
logn_data[n] = Math.log10(n) * logn_sample
|
115
|
-
n_data[n] = n * n_sample
|
116
|
-
nlogn_data[n] = n * Math.log10(n) * nlogn_sample
|
117
|
-
n2_data[n] = n * n * n2_sample
|
118
|
-
end
|
119
|
-
|
120
|
-
comparison_data = []
|
121
|
-
comparison_data << chart_data
|
122
|
-
comparison_data << {name: 'log n', data: logn_data}
|
123
|
-
comparison_data << {name: 'n', data: n_data}
|
124
|
-
comparison_data << {name: 'n log n', data: nlogn_data}
|
125
|
-
comparison_data << {name: 'n_sq', data: n2_data}
|
126
|
-
comparison_data
|
127
|
-
end
|
128
44
|
end
|
129
45
|
end
|
130
46
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
|
3
|
+
class Benchmark::BigO::TermPlot
|
4
|
+
|
5
|
+
def initialize report_data, sizes
|
6
|
+
@data = report_data
|
7
|
+
@sizes = sizes
|
8
|
+
@dat_file = Tempfile.new 'termplot'
|
9
|
+
end
|
10
|
+
|
11
|
+
def generate
|
12
|
+
|
13
|
+
write_dat_file
|
14
|
+
|
15
|
+
begin
|
16
|
+
IO.popen("gnuplot", "w") { |io| io.puts commands }
|
17
|
+
rescue
|
18
|
+
puts "You need to have gnuplot installed!"
|
19
|
+
puts "brew install gnuplot"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def write_dat_file
|
24
|
+
File.open(@dat_file.path, 'w') do |file|
|
25
|
+
dat_lines.each do |line|
|
26
|
+
file.puts line
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def dat_lines
|
32
|
+
lines = []
|
33
|
+
@sizes.each do |size|
|
34
|
+
line = [size] + @data.collect{|d| d[:data][size] }
|
35
|
+
lines << line.join(' ')
|
36
|
+
end
|
37
|
+
lines
|
38
|
+
end
|
39
|
+
|
40
|
+
def title_for index
|
41
|
+
@data[index][:name]
|
42
|
+
end
|
43
|
+
|
44
|
+
def commands
|
45
|
+
|
46
|
+
cmds = []
|
47
|
+
cmds << "set term dumb"
|
48
|
+
cmds << "set key top left vertical inside width 3"
|
49
|
+
|
50
|
+
data_idx = 0
|
51
|
+
|
52
|
+
while data_idx < @data.length
|
53
|
+
|
54
|
+
if data_idx == 0
|
55
|
+
str = "plot '#{@dat_file.path}'"
|
56
|
+
else
|
57
|
+
str = "\"\""
|
58
|
+
end
|
59
|
+
|
60
|
+
str += " using 1:#{data_idx+2} title '#{title_for(data_idx)}' with lines"
|
61
|
+
str += ",\\" unless data_idx == @data.length-1
|
62
|
+
|
63
|
+
cmds << str
|
64
|
+
|
65
|
+
data_idx += 1
|
66
|
+
end
|
67
|
+
|
68
|
+
cmds
|
69
|
+
end
|
70
|
+
end
|
data/test/benchmark/test_bigo.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require "minitest/autorun"
|
2
2
|
require "benchmark/bigo"
|
3
|
+
require "tempfile"
|
3
4
|
|
4
5
|
class TestBenchmarkBigo < MiniTest::Test
|
5
6
|
def setup
|
@@ -11,146 +12,237 @@ class TestBenchmarkBigo < MiniTest::Test
|
|
11
12
|
$stdout = @old_stdout
|
12
13
|
end
|
13
14
|
|
14
|
-
def
|
15
|
-
|
15
|
+
def test_bigo_defaults
|
16
|
+
report = Benchmark.bigo do |x|
|
17
|
+
x.config(:time => 1, :warmup => 1)
|
18
|
+
x.generate :array
|
19
|
+
|
20
|
+
# size 100 will sleep for .1 seconds, size 200 will sleep for .2 seconds
|
21
|
+
x.report("sleep") { |_, size| sleep(size / 1000.0) }
|
22
|
+
end
|
23
|
+
|
24
|
+
assert_equal 1, report.entries.keys.length
|
25
|
+
rep = report.entries["sleep"]
|
26
|
+
|
27
|
+
assert_equal 10, rep.size
|
28
|
+
|
29
|
+
10.times do |i|
|
30
|
+
size = 100 + i*100
|
31
|
+
assert_equal "sleep #{size}", rep[i].label
|
32
|
+
|
33
|
+
iterations = 1000.0 / size
|
34
|
+
assert_equal iterations.ceil, rep[i].iterations
|
35
|
+
assert_in_delta iterations, rep[i].ips, 0.4
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_bigo_config_options
|
16
40
|
|
17
41
|
report = Benchmark.bigo do |x|
|
18
|
-
x.config(:time => 1, :warmup => 1,
|
42
|
+
x.config(:time => 1, :warmup => 1,
|
43
|
+
:steps => 3, :step_size => 200, :min_size => 50)
|
19
44
|
x.generate :array
|
20
45
|
|
21
46
|
x.report("#at") {|array, size| array.at rand(size) }
|
22
|
-
|
23
|
-
|
47
|
+
|
48
|
+
# size 100 will sleep for .1 seconds, size 200 will sleep for .2 seconds
|
49
|
+
x.report("sleep") { |_, size| sleep(size / 1000.0) }
|
24
50
|
end
|
25
51
|
|
26
|
-
assert_equal
|
52
|
+
assert_equal 2, report.entries.length
|
27
53
|
|
28
54
|
assert report.entries.keys.include?("#at")
|
29
|
-
assert report.entries.keys.include?("#index")
|
30
55
|
assert report.entries.keys.include?("sleep")
|
31
56
|
|
32
57
|
at_rep = report.entries["#at"]
|
33
|
-
index_rep = report.entries["#index"]
|
34
58
|
sleep_rep = report.entries["sleep"]
|
35
59
|
|
36
|
-
assert_equal
|
37
|
-
assert_equal
|
38
|
-
assert_equal 2, sleep_rep.size
|
60
|
+
assert_equal 3, at_rep.size
|
61
|
+
assert_equal 3, sleep_rep.size
|
39
62
|
|
40
|
-
assert_equal "#at
|
41
|
-
assert_equal "#at
|
63
|
+
assert_equal "#at 50", at_rep[0].label
|
64
|
+
assert_equal "#at 250", at_rep[1].label
|
65
|
+
assert_equal "#at 450", at_rep[2].label
|
42
66
|
|
43
|
-
assert_equal "
|
44
|
-
assert_equal "
|
67
|
+
assert_equal "sleep 50", sleep_rep[0].label
|
68
|
+
assert_equal "sleep 250", sleep_rep[1].label
|
69
|
+
assert_equal "sleep 450", sleep_rep[2].label
|
45
70
|
|
46
|
-
assert_equal
|
47
|
-
|
71
|
+
assert_equal 20, sleep_rep[0].iterations
|
72
|
+
assert_in_delta 20.0, sleep_rep[0].ips, 0.6
|
48
73
|
|
49
|
-
assert_equal 4, sleep_rep[
|
50
|
-
assert_in_delta 4.0, sleep_rep[
|
74
|
+
assert_equal 4, sleep_rep[1].iterations
|
75
|
+
assert_in_delta 4.0, sleep_rep[1].ips, 0.2
|
51
76
|
|
52
77
|
end
|
53
78
|
|
54
|
-
def
|
55
|
-
#skip('pass')
|
79
|
+
def test_bigo_setters
|
56
80
|
report = Benchmark.bigo do |x|
|
57
81
|
x.time = 1
|
58
82
|
x.warmup = 1
|
59
|
-
x.
|
60
|
-
x.
|
61
|
-
x.
|
83
|
+
x.steps = 3
|
84
|
+
x.step_size = 200
|
85
|
+
x.min_size = 50
|
86
|
+
x.generate :array
|
87
|
+
|
88
|
+
x.report("#at") {|array, size| array.at rand(size) }
|
89
|
+
|
90
|
+
# size 100 will sleep for .1 seconds, size 200 will sleep for .2 seconds
|
91
|
+
x.report("sleep") { |_, size| sleep(size / 1000.0) }
|
62
92
|
end
|
63
93
|
|
64
|
-
|
65
|
-
assert_equal 2, rep.size
|
94
|
+
assert_equal 2, report.entries.length
|
66
95
|
|
67
|
-
|
68
|
-
|
69
|
-
assert_in_delta 4.0, rep[0].ips, 0.2
|
70
|
-
end
|
96
|
+
assert report.entries.keys.include?("#at")
|
97
|
+
assert report.entries.keys.include?("sleep")
|
71
98
|
|
72
|
-
|
73
|
-
|
74
|
-
report = Benchmark.bigo do |x|
|
75
|
-
x.config(:time => 1, :warmup => 1)
|
76
|
-
x.generate :array
|
77
|
-
x.report("sleep") { |a,b| sleep(0.25) }
|
78
|
-
end
|
99
|
+
at_rep = report.entries["#at"]
|
100
|
+
sleep_rep = report.entries["sleep"]
|
79
101
|
|
80
|
-
assert_equal
|
81
|
-
|
102
|
+
assert_equal 3, at_rep.size
|
103
|
+
assert_equal 3, sleep_rep.size
|
82
104
|
|
83
|
-
assert_equal
|
105
|
+
assert_equal "#at 50", at_rep[0].label
|
106
|
+
assert_equal "#at 250", at_rep[1].label
|
107
|
+
assert_equal "#at 450", at_rep[2].label
|
84
108
|
|
85
|
-
assert_equal "sleep
|
86
|
-
assert_equal "sleep
|
87
|
-
assert_equal "sleep
|
88
|
-
assert_equal "sleep 400", rep[3].label
|
89
|
-
assert_equal "sleep 500", rep[4].label
|
109
|
+
assert_equal "sleep 50", sleep_rep[0].label
|
110
|
+
assert_equal "sleep 250", sleep_rep[1].label
|
111
|
+
assert_equal "sleep 450", sleep_rep[2].label
|
90
112
|
|
91
|
-
assert_equal
|
92
|
-
assert_in_delta
|
113
|
+
assert_equal 20, sleep_rep[0].iterations
|
114
|
+
assert_in_delta 20.0, sleep_rep[0].ips, 0.6
|
115
|
+
|
116
|
+
assert_equal 4, sleep_rep[1].iterations
|
117
|
+
assert_in_delta 4.0, sleep_rep[1].ips, 0.2
|
93
118
|
end
|
94
119
|
|
95
|
-
def
|
96
|
-
|
120
|
+
def test_bigo_generate_json
|
121
|
+
json_file = Tempfile.new 'data.json'
|
97
122
|
report = Benchmark.bigo do |x|
|
98
|
-
x.config(:time => 1, :warmup => 1, :
|
123
|
+
x.config(:time => 1, :warmup => 1, :steps => 2)
|
99
124
|
x.generate :array
|
100
|
-
|
101
|
-
x.report("
|
125
|
+
|
126
|
+
x.report("#at") {|array, size| array.at rand(size) }
|
127
|
+
x.json! json_file.path
|
102
128
|
end
|
103
129
|
|
104
|
-
|
130
|
+
json_data = json_file.read
|
131
|
+
assert json_data
|
132
|
+
data = JSON.parse json_data
|
133
|
+
assert data
|
134
|
+
assert_equal 1, data.size
|
135
|
+
assert_equal "#at", data[0]["name"]
|
136
|
+
assert data[0]["data"]
|
137
|
+
assert data[0]["data"]["100"]
|
138
|
+
assert data[0]["data"]["200"]
|
139
|
+
end
|
105
140
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
141
|
+
def test_bigo_generate_csv
|
142
|
+
csv_file = Tempfile.new 'data.csv'
|
143
|
+
report = Benchmark.bigo do |x|
|
144
|
+
x.config(:time => 1, :warmup => 1, :steps => 2)
|
145
|
+
x.generate :array
|
146
|
+
|
147
|
+
x.report("#at") {|array, size| array.at rand(size) }
|
148
|
+
x.csv! csv_file.path
|
149
|
+
end
|
150
|
+
data = CSV.read(csv_file.path)
|
151
|
+
assert data
|
152
|
+
assert_equal 2, data.size
|
153
|
+
assert_equal ['','100', '200'], data[0]
|
154
|
+
assert_equal "#at", data[1][0]
|
155
|
+
assert_equal data[1][0].size, 3
|
110
156
|
end
|
111
157
|
|
112
|
-
def
|
113
|
-
|
158
|
+
def test_bigo_generate_chart
|
159
|
+
chart_file = Tempfile.new 'data.html'
|
114
160
|
report = Benchmark.bigo do |x|
|
115
|
-
x.config(:time => 1, :warmup => 1, :
|
161
|
+
x.config(:time => 1, :warmup => 1, :steps => 2)
|
116
162
|
x.generate :array
|
117
163
|
|
118
164
|
x.report("#at") {|array, size| array.at rand(size) }
|
165
|
+
x.chart! chart_file.path
|
166
|
+
x.compare!
|
119
167
|
end
|
120
168
|
|
121
|
-
|
169
|
+
data = File.read(chart_file.path)
|
170
|
+
assert data
|
171
|
+
assert data.match('<h1>Growth Chart</h1>')
|
172
|
+
assert data.match('<h1>#at</h1>')
|
173
|
+
end
|
174
|
+
|
175
|
+
def test_generate_array
|
122
176
|
|
123
|
-
|
177
|
+
job = Benchmark::BigO::Job.new({:suite => nil,
|
178
|
+
:quiet => true})
|
124
179
|
|
125
|
-
|
180
|
+
job.generate :array
|
126
181
|
|
127
|
-
|
182
|
+
job.report('test') {|array, size| array.at rand(size) }
|
183
|
+
|
184
|
+
# should create 10 job entries
|
185
|
+
assert_equal 10, job.list.size
|
128
186
|
|
129
|
-
|
130
|
-
|
131
|
-
|
187
|
+
# the generated object should be an Array
|
188
|
+
assert Array === job.list.first.generated
|
189
|
+
|
190
|
+
# the Array should have 100 elements
|
191
|
+
assert_equal 100, job.list.first.generated.size
|
192
|
+
|
193
|
+
# when sorted, the array should equal 0...100
|
194
|
+
assert_equal (0...100).to_a, job.list.first.generated.sort
|
132
195
|
end
|
133
196
|
|
134
|
-
def
|
135
|
-
#skip('pass')
|
136
|
-
report = Benchmark.bigo do |x|
|
137
|
-
x.config(:time => 1, :warmup => 1)
|
138
|
-
x.increments = 3
|
139
|
-
x.generate :array
|
197
|
+
def test_generate_string
|
140
198
|
|
141
|
-
|
142
|
-
|
199
|
+
job = Benchmark::BigO::Job.new({:suite => nil,
|
200
|
+
:quiet => true})
|
143
201
|
|
144
|
-
|
202
|
+
job.generate :string
|
145
203
|
|
146
|
-
|
204
|
+
job.report('test') {|string, size| string[rand(size)] }
|
147
205
|
|
148
|
-
|
206
|
+
# should create 10 job entries
|
207
|
+
assert_equal 10, job.list.size
|
149
208
|
|
150
|
-
|
209
|
+
# the generated object should be a String
|
210
|
+
assert String === job.list.first.generated
|
211
|
+
|
212
|
+
# the String should have length 100
|
213
|
+
assert_equal 200, job.list.first.generated.length
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_generate_size
|
217
|
+
job = Benchmark::BigO::Job.new({:suite => nil,
|
218
|
+
:quiet => true})
|
219
|
+
|
220
|
+
job.generate :size
|
221
|
+
|
222
|
+
job.report('test') {|size, _| size.to_i }
|
223
|
+
|
224
|
+
# should create 10 job entries
|
225
|
+
assert_equal 10, job.list.size
|
226
|
+
|
227
|
+
# the generated object should be an Integer
|
228
|
+
assert Integer === job.list.first.generated
|
229
|
+
|
230
|
+
# the Integer should be the size
|
231
|
+
assert_equal 100, job.list.first.generated
|
232
|
+
assert_equal 1000, job.list[-1].generated
|
233
|
+
|
234
|
+
end
|
235
|
+
|
236
|
+
def test_step
|
237
|
+
job = Benchmark::BigO::Job.new({:suite => nil,
|
238
|
+
:quiet => true})
|
239
|
+
|
240
|
+
job.min_size = 22
|
241
|
+
job.steps = 6
|
242
|
+
job.step_size = 10
|
151
243
|
|
152
|
-
assert_equal
|
153
|
-
assert_equal
|
154
|
-
assert_equal
|
244
|
+
assert_equal 22, job.step(0)
|
245
|
+
assert_equal 32, job.step(1)
|
246
|
+
assert_equal 42, job.step(2)
|
155
247
|
end
|
156
248
|
end
|