benchcc 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/benchcc/benchmark.rb +29 -23
- data/lib/benchcc/compiler.rb +68 -116
- data/lib/benchcc/plot.rb +29 -26
- data/lib/benchcc/version.rb +1 -1
- data/spec/benchmark_spec.rb +26 -31
- data/spec/compiler_spec.rb +4 -26
- data/spec/plot_spec.rb +8 -10
- data/spec/src/plot.csv +3 -3
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6565e1423e649ab808388cd0d7fe2ce78fc8bf38
|
4
|
+
data.tar.gz: 8b68dc1cdf7e37e92a76731cf4e3407fcbe11e83
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc94f060eb50fb5cdfd2e325fdfbf21bdb0215ce790b6706a33f34c4872834ab2a9eaaa8fc8e157bdec351d6cc35833d9c4c479c7cb2efcd098df3cb40e449e5
|
7
|
+
data.tar.gz: 52b0e9d48f49151e1563859b1709165217e9e4edc12ce194dfcc4719d7885c8843fc4f8502a1f1cdc992d2b34781376bf47331970bcddd2ec87d926af4a2bb75
|
data/lib/benchcc/benchmark.rb
CHANGED
@@ -20,34 +20,40 @@ module Benchcc
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def benchmark(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
&block)
|
28
|
-
progress = ProgressBar.create(format: progressbar_format, total: envs.size)
|
29
|
-
data = []
|
30
|
-
envs.each do |env|
|
31
|
-
code = Renderer.new(relative_to).render(file, **env)
|
32
|
-
begin
|
33
|
-
Timeout::timeout(timeout) { data << env.merge(block.call(code).to_h) }
|
34
|
-
rescue CompilationError, Timeout::Error => e
|
35
|
-
stderr << e
|
36
|
-
break
|
37
|
-
end
|
38
|
-
progress.increment
|
39
|
-
end
|
23
|
+
def benchmark(erb_file, environments, timeout: 10,
|
24
|
+
evaluate_erb_relative_to: File.dirname(erb_file), &bench)
|
25
|
+
erb_file = Pathname.new(erb_file)
|
26
|
+
progress = ProgressBar.create(format: '%p%% | %B |', total: environments.size)
|
40
27
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
28
|
+
data = CSV.generate({headers: :first_row}) do |csv|
|
29
|
+
csv << [:input_size, :compilation_time, :memory_usage, :run_time]
|
30
|
+
|
31
|
+
environments.each do |env|
|
32
|
+
code = Renderer.new(evaluate_erb_relative_to).render(erb_file, **env)
|
33
|
+
begin
|
34
|
+
Tempfile.create([erb_file.basename, '.cpp']) do |tmp|
|
35
|
+
tmp.write(code) && tmp.close
|
36
|
+
bench_data = Timeout::timeout(timeout) {
|
37
|
+
bench.call(tmp.path, env)
|
38
|
+
}
|
39
|
+
row = {
|
40
|
+
input_size: env[:input_size],
|
41
|
+
compilation_time: bench_data[:compilation_time],
|
42
|
+
memory_usage: bench_data[:memory_usage],
|
43
|
+
run_time: bench_data[:run_time] # TODO: implement this
|
44
|
+
}
|
45
|
+
csv << row
|
46
|
+
end
|
47
|
+
rescue CompilationError, Timeout::Error => e
|
48
|
+
$stderr << e
|
49
|
+
break
|
50
|
+
end
|
51
|
+
progress.increment
|
45
52
|
end
|
46
53
|
end
|
47
|
-
|
48
54
|
return data
|
49
55
|
ensure
|
50
|
-
|
56
|
+
progress.finish
|
51
57
|
end
|
52
58
|
module_function :benchmark
|
53
59
|
end
|
data/lib/benchcc/compiler.rb
CHANGED
@@ -7,79 +7,34 @@ module Benchcc
|
|
7
7
|
class CompilationError < RuntimeError
|
8
8
|
end
|
9
9
|
|
10
|
-
# Structure holding various information about the compilation of a file.
|
11
|
-
CompilationResult = Struct.new('CompilationResult',
|
12
|
-
# The time taken to compile.
|
13
|
-
:wall_time,
|
14
|
-
# The peak memory usage during compilation.
|
15
|
-
:peak_memusg,
|
16
|
-
# The standard error produced during compilation.
|
17
|
-
:stderr,
|
18
|
-
# The standard output produced during compilation.
|
19
|
-
:stdout,
|
20
|
-
# The input source that was compiled.
|
21
|
-
:code,
|
22
|
-
# The command line used for compilation.
|
23
|
-
:command_line
|
24
|
-
) do
|
25
|
-
def to_s
|
26
|
-
<<-EOS
|
27
|
-
> #{command_line}
|
28
|
-
#{stderr}
|
29
|
-
|
30
|
-
[compiling:
|
31
|
-
#{code}
|
32
|
-
]
|
33
|
-
EOS
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
10
|
# Basic interface to compiler frontends.
|
38
11
|
class Compiler
|
39
|
-
# Maximum template recursion depth supported by the compiler.
|
40
|
-
def template_depth
|
41
|
-
raise NotImplementedError
|
42
|
-
end
|
43
|
-
|
44
|
-
# Maximum constexpr recursion depth supported by the compiler.
|
45
|
-
def constexpr_depth
|
46
|
-
raise NotImplementedError
|
47
|
-
end
|
48
|
-
|
49
12
|
# Show the name and the version of the compiler.
|
50
13
|
def to_s
|
51
14
|
raise NotImplementedError
|
52
15
|
end
|
53
16
|
|
54
|
-
#
|
17
|
+
# compile: Path -> Hash
|
55
18
|
#
|
56
19
|
# Compile the given file and return compilation statistics.
|
57
20
|
#
|
58
|
-
# Additional compiler-specific arguments may be specified.
|
59
|
-
#
|
60
|
-
#
|
61
|
-
|
62
|
-
|
63
|
-
def compile_file(file, *args)
|
64
|
-
raise ArgumentError, "invalid filename #{file}" unless File.file? file
|
65
|
-
code = File.read(file)
|
66
|
-
compile_code(code, *args)
|
21
|
+
# Additional compiler-specific arguments may be specified. A
|
22
|
+
# `CompilationError` is raised if the compilation fails for
|
23
|
+
# whatever reason.
|
24
|
+
def compile(file, *args)
|
25
|
+
raise NotImplementedError
|
67
26
|
end
|
68
27
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
tmp.close
|
80
|
-
compile_file(tmp.path, *args)
|
81
|
-
ensure
|
82
|
-
tmp.unlink
|
28
|
+
def self.guess_from_binary(binary)
|
29
|
+
stdout, stderr, status = Open3.capture3("#{binary} --version")
|
30
|
+
case stdout
|
31
|
+
when /\(GCC\)/
|
32
|
+
return GCC.new(binary)
|
33
|
+
when /clang/
|
34
|
+
return Clang.new(binary)
|
35
|
+
else
|
36
|
+
raise ArgumentError("unknown compiler #{binary}")
|
37
|
+
end
|
83
38
|
end
|
84
39
|
end # class Compiler
|
85
40
|
|
@@ -89,41 +44,47 @@ EOS
|
|
89
44
|
raise "#{binary} not found" unless $?.success?
|
90
45
|
end
|
91
46
|
|
92
|
-
def
|
47
|
+
def compile(file, *args)
|
93
48
|
file = Pathname.new(file).expand_path
|
94
49
|
command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
|
95
50
|
stdout, stderr, status = Open3.capture3(command)
|
96
51
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
52
|
+
if status.success?
|
53
|
+
memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
54
|
+
|
55
|
+
section = -> (title) {
|
56
|
+
title.gsub!(' ', '\s')
|
57
|
+
/(
|
58
|
+
===-+===\n
|
59
|
+
.*#{title}.*\n
|
60
|
+
===-+===\n
|
61
|
+
(.|\n)+?(?====)
|
62
|
+
)|(
|
63
|
+
===-+===\n
|
64
|
+
.*#{title}.*\n
|
65
|
+
===-+===\n
|
66
|
+
(.|\n)+
|
67
|
+
)/x
|
68
|
+
}
|
69
|
+
time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
|
70
|
+
.match(/(\d+\.?\d+).+?Total/)[1]
|
71
|
+
return {
|
72
|
+
compilation_time: time,
|
73
|
+
memory_usage: memusg
|
74
|
+
}
|
103
75
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
)
|
114
|
-
|
115
|
-
.*#{title}.*\n
|
116
|
-
===-+===\n
|
117
|
-
(.|\n)+
|
118
|
-
)/x
|
119
|
-
}
|
120
|
-
result.wall_time = stderr.match(section["Miscellaneous Ungrouped Timers"])
|
121
|
-
.to_s.match(/(\d+\.?\d+).+?Total/)[1]
|
122
|
-
return result
|
76
|
+
else
|
77
|
+
err_string = <<-EOS
|
78
|
+
> #{command}
|
79
|
+
#{stderr}
|
80
|
+
|
81
|
+
[compiling:
|
82
|
+
#{file.read}
|
83
|
+
]
|
84
|
+
EOS
|
85
|
+
raise CompilationError.new(err_string)
|
86
|
+
end
|
123
87
|
end
|
124
|
-
|
125
|
-
def template_depth; 256; end
|
126
|
-
def constexpr_depth; 512; end
|
127
88
|
end
|
128
89
|
|
129
90
|
class GCC < Compiler
|
@@ -132,37 +93,28 @@ EOS
|
|
132
93
|
raise "#{binary} not found" unless $?.success?
|
133
94
|
end
|
134
95
|
|
135
|
-
def
|
96
|
+
def compile(file, *args)
|
136
97
|
file = Pathname.new(file).expand_path
|
137
98
|
command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
|
138
99
|
stdout, stderr, status = Open3.capture3(command)
|
139
100
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
148
|
-
result.wall_time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
|
149
|
-
return result
|
150
|
-
end
|
151
|
-
|
152
|
-
def template_depth; 900; end
|
153
|
-
def constexpr_depth; 512; end
|
154
|
-
end
|
155
|
-
|
156
|
-
class Compiler
|
157
|
-
def self.guess_from_binary(binary)
|
158
|
-
stdout, stderr, status = Open3.capture3("#{binary} --version")
|
159
|
-
case stdout
|
160
|
-
when /\(GCC\)/
|
161
|
-
return GCC.new(binary)
|
162
|
-
when /clang/
|
163
|
-
return Clang.new(binary)
|
101
|
+
if status.success?
|
102
|
+
time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
|
103
|
+
memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
104
|
+
return {
|
105
|
+
compilation_time: time,
|
106
|
+
memory_usage: memusg
|
107
|
+
}
|
164
108
|
else
|
165
|
-
|
109
|
+
err_string = <<-EOS
|
110
|
+
> #{command}
|
111
|
+
#{stderr}
|
112
|
+
|
113
|
+
[compiling:
|
114
|
+
#{file.read}
|
115
|
+
]
|
116
|
+
EOS
|
117
|
+
raise CompilationError.new(err_string)
|
166
118
|
end
|
167
119
|
end
|
168
120
|
end
|
data/lib/benchcc/plot.rb
CHANGED
@@ -3,42 +3,45 @@ require 'gnuplot'
|
|
3
3
|
|
4
4
|
|
5
5
|
module Benchcc
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
6
|
+
Y_FEATURES = [:memory_usage, :compilation_time, :run_time]
|
7
|
+
|
8
|
+
# How do I make this private?
|
9
|
+
DEFAULT_TWEAK = {
|
10
|
+
memory_usage: -> (plot) {
|
11
|
+
plot.ylabel 'Memory usage'
|
12
|
+
plot.decimal 'locale \'en_US.UTF-8\''
|
13
|
+
plot.format 'y \'%.2e kb\''
|
14
|
+
},
|
15
|
+
|
16
|
+
compilation_time: -> (plot) {
|
17
|
+
plot.ylabel 'Compilation time'
|
18
|
+
plot.format 'y \'%.2g s\''
|
19
|
+
},
|
20
|
+
|
21
|
+
run_time: -> (plot) {
|
22
|
+
plot.ylabel 'Run time'
|
23
|
+
plot.format 'y \'%.2g s\''
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
def plot(output, titles, inputs, x_feature: :input_size, y_feature: :compilation_time, &tweak)
|
28
|
+
raise ArgumentError if not Benchcc::Y_FEATURES.include?(y_feature)
|
29
|
+
tweak ||= Benchcc::DEFAULT_TWEAK[y_feature]
|
25
30
|
|
26
|
-
def plot_time(output, *inputs)
|
27
31
|
Gnuplot.open do |io|
|
28
32
|
Gnuplot::Plot.new(io) do |plot|
|
29
|
-
plot.ylabel 'Compilation time'
|
30
|
-
plot.format 'y "%.2g s"'
|
31
33
|
plot.term 'png'
|
32
34
|
plot.output output
|
33
|
-
plot.data = inputs.map { |file|
|
35
|
+
plot.data = titles.zip(inputs).map { |title, file|
|
34
36
|
csv = CSV.table(file)
|
35
|
-
Gnuplot::DataSet.new([csv[
|
36
|
-
ds.title =
|
37
|
+
Gnuplot::DataSet.new([csv[x_feature], csv[y_feature]]) { |ds|
|
38
|
+
ds.title = title
|
37
39
|
ds.with = 'lines'
|
38
40
|
}
|
39
41
|
}
|
42
|
+
tweak.call(plot)
|
40
43
|
end
|
41
44
|
end
|
42
45
|
end
|
43
|
-
module_function :
|
46
|
+
module_function :plot
|
44
47
|
end
|
data/lib/benchcc/version.rb
CHANGED
data/spec/benchmark_spec.rb
CHANGED
@@ -1,50 +1,45 @@
|
|
1
1
|
require 'benchcc/benchmark'
|
2
|
+
require 'benchcc/compiler'
|
2
3
|
|
3
4
|
require 'rspec'
|
4
5
|
require 'tempfile'
|
5
|
-
require 'timeout'
|
6
6
|
|
7
7
|
|
8
8
|
describe Benchcc.method(:benchmark) do
|
9
|
-
it('should
|
10
|
-
|
11
|
-
envs = [{time: timeout - 0.1}, {time: timeout + 0.1}, {time: timeout - 0.2}]
|
9
|
+
it('should not throw') {
|
10
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
12
11
|
Tempfile.create('') do |file|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
12
|
+
expect {
|
13
|
+
data = Benchcc.benchmark(file.path, envs) do |file, env|
|
14
|
+
{
|
15
|
+
compilation_time: 'foo',
|
16
|
+
memory_usage: 'bar',
|
17
|
+
run_time: 'baz'
|
18
|
+
}
|
19
|
+
end
|
20
|
+
}.not_to raise_error
|
21
21
|
end
|
22
22
|
}
|
23
23
|
|
24
|
-
it('should
|
25
|
-
envs = [{
|
24
|
+
it('should stop on timeout') {
|
25
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
26
26
|
Tempfile.create('') do |file|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
expect(data).to eq(envs)
|
27
|
+
expect {
|
28
|
+
data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
|
29
|
+
sleep(0.2)
|
30
|
+
end
|
31
|
+
}.not_to raise_error
|
33
32
|
end
|
34
33
|
}
|
35
|
-
end
|
36
34
|
|
37
|
-
|
38
|
-
|
39
|
-
envs = [{x: 1, y:-1}, {x: 2, y:-2}, {x: 3, y:-3}]
|
35
|
+
it('should stop on CompilationError') {
|
36
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
40
37
|
Tempfile.create('') do |file|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
}.to_csv
|
47
|
-
expect(csv).to eq("x,y\n1,-1\n2,-2\n3,-3\n")
|
38
|
+
expect {
|
39
|
+
data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
|
40
|
+
raise Benchcc::CompilationError.new
|
41
|
+
end
|
42
|
+
}.not_to raise_error
|
48
43
|
end
|
49
44
|
}
|
50
45
|
end
|
data/spec/compiler_spec.rb
CHANGED
@@ -6,28 +6,6 @@ require 'tempfile'
|
|
6
6
|
|
7
7
|
|
8
8
|
describe Benchcc::Compiler do
|
9
|
-
describe :compile_file do
|
10
|
-
it {
|
11
|
-
class Mock < Benchcc::Compiler
|
12
|
-
def compile_code(code); code; end
|
13
|
-
end
|
14
|
-
|
15
|
-
file = Tempfile.new('')
|
16
|
-
file.write("whatever")
|
17
|
-
expect(Mock.new.compile_file(file.path)).to eq(file.read)
|
18
|
-
}
|
19
|
-
end
|
20
|
-
|
21
|
-
describe :compile_code do
|
22
|
-
it {
|
23
|
-
class Mock < Benchcc::Compiler
|
24
|
-
def compile_file(file); File.read(file); end
|
25
|
-
end
|
26
|
-
|
27
|
-
expect(Mock.new.compile_code("whatever")).to eq("whatever")
|
28
|
-
}
|
29
|
-
end
|
30
|
-
|
31
9
|
describe :guess_from_binary do
|
32
10
|
it('can guess clang') {
|
33
11
|
expect(
|
@@ -58,17 +36,17 @@ end
|
|
58
36
|
@valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
|
59
37
|
}
|
60
38
|
|
61
|
-
describe :
|
39
|
+
describe :compile do
|
62
40
|
it('fails cleanly on invalid input') {
|
63
41
|
expect {
|
64
|
-
@cc.
|
42
|
+
@cc.compile(@invalid)
|
65
43
|
}.to raise_error(Benchcc::CompilationError)
|
66
44
|
}
|
67
45
|
|
68
46
|
it('returns statistics on valid input') {
|
69
47
|
expect(
|
70
|
-
@cc.
|
71
|
-
).to be_instance_of(
|
48
|
+
@cc.compile(@valid, '-o /dev/null')
|
49
|
+
).to be_instance_of(Hash)
|
72
50
|
}
|
73
51
|
end
|
74
52
|
end
|
data/spec/plot_spec.rb
CHANGED
@@ -15,15 +15,13 @@ describe 'plots' do
|
|
15
15
|
FileUtils.remove_entry(@tmpdir)
|
16
16
|
end
|
17
17
|
|
18
|
-
describe Benchcc.method(:
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
expect { Benchcc.plot_time(@out, @csv) }.not_to raise_error
|
27
|
-
}
|
18
|
+
describe Benchcc.method(:plot) do
|
19
|
+
Benchcc::Y_FEATURES.each do |feature|
|
20
|
+
it {
|
21
|
+
expect {
|
22
|
+
Benchcc.plot(@out, [''], [@csv], y_feature: feature)
|
23
|
+
}.not_to raise_error
|
24
|
+
}
|
25
|
+
end
|
28
26
|
end
|
29
27
|
end
|
data/spec/src/plot.csv
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
input_size, memory_usage, compilation_time, run_time
|
2
|
+
1, 2, 3, 4
|
3
|
+
5, 6, 7, 8
|