benchcc 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/benchcc/benchmark.rb +29 -23
- data/lib/benchcc/compiler.rb +68 -116
- data/lib/benchcc/plot.rb +29 -26
- data/lib/benchcc/version.rb +1 -1
- data/spec/benchmark_spec.rb +26 -31
- data/spec/compiler_spec.rb +4 -26
- data/spec/plot_spec.rb +8 -10
- data/spec/src/plot.csv +3 -3
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6565e1423e649ab808388cd0d7fe2ce78fc8bf38
|
4
|
+
data.tar.gz: 8b68dc1cdf7e37e92a76731cf4e3407fcbe11e83
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc94f060eb50fb5cdfd2e325fdfbf21bdb0215ce790b6706a33f34c4872834ab2a9eaaa8fc8e157bdec351d6cc35833d9c4c479c7cb2efcd098df3cb40e449e5
|
7
|
+
data.tar.gz: 52b0e9d48f49151e1563859b1709165217e9e4edc12ce194dfcc4719d7885c8843fc4f8502a1f1cdc992d2b34781376bf47331970bcddd2ec87d926af4a2bb75
|
data/lib/benchcc/benchmark.rb
CHANGED
@@ -20,34 +20,40 @@ module Benchcc
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def benchmark(
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
&block)
|
28
|
-
progress = ProgressBar.create(format: progressbar_format, total: envs.size)
|
29
|
-
data = []
|
30
|
-
envs.each do |env|
|
31
|
-
code = Renderer.new(relative_to).render(file, **env)
|
32
|
-
begin
|
33
|
-
Timeout::timeout(timeout) { data << env.merge(block.call(code).to_h) }
|
34
|
-
rescue CompilationError, Timeout::Error => e
|
35
|
-
stderr << e
|
36
|
-
break
|
37
|
-
end
|
38
|
-
progress.increment
|
39
|
-
end
|
23
|
+
def benchmark(erb_file, environments, timeout: 10,
|
24
|
+
evaluate_erb_relative_to: File.dirname(erb_file), &bench)
|
25
|
+
erb_file = Pathname.new(erb_file)
|
26
|
+
progress = ProgressBar.create(format: '%p%% | %B |', total: environments.size)
|
40
27
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
28
|
+
data = CSV.generate({headers: :first_row}) do |csv|
|
29
|
+
csv << [:input_size, :compilation_time, :memory_usage, :run_time]
|
30
|
+
|
31
|
+
environments.each do |env|
|
32
|
+
code = Renderer.new(evaluate_erb_relative_to).render(erb_file, **env)
|
33
|
+
begin
|
34
|
+
Tempfile.create([erb_file.basename, '.cpp']) do |tmp|
|
35
|
+
tmp.write(code) && tmp.close
|
36
|
+
bench_data = Timeout::timeout(timeout) {
|
37
|
+
bench.call(tmp.path, env)
|
38
|
+
}
|
39
|
+
row = {
|
40
|
+
input_size: env[:input_size],
|
41
|
+
compilation_time: bench_data[:compilation_time],
|
42
|
+
memory_usage: bench_data[:memory_usage],
|
43
|
+
run_time: bench_data[:run_time] # TODO: implement this
|
44
|
+
}
|
45
|
+
csv << row
|
46
|
+
end
|
47
|
+
rescue CompilationError, Timeout::Error => e
|
48
|
+
$stderr << e
|
49
|
+
break
|
50
|
+
end
|
51
|
+
progress.increment
|
45
52
|
end
|
46
53
|
end
|
47
|
-
|
48
54
|
return data
|
49
55
|
ensure
|
50
|
-
|
56
|
+
progress.finish
|
51
57
|
end
|
52
58
|
module_function :benchmark
|
53
59
|
end
|
data/lib/benchcc/compiler.rb
CHANGED
@@ -7,79 +7,34 @@ module Benchcc
|
|
7
7
|
class CompilationError < RuntimeError
|
8
8
|
end
|
9
9
|
|
10
|
-
# Structure holding various information about the compilation of a file.
|
11
|
-
CompilationResult = Struct.new('CompilationResult',
|
12
|
-
# The time taken to compile.
|
13
|
-
:wall_time,
|
14
|
-
# The peak memory usage during compilation.
|
15
|
-
:peak_memusg,
|
16
|
-
# The standard error produced during compilation.
|
17
|
-
:stderr,
|
18
|
-
# The standard output produced during compilation.
|
19
|
-
:stdout,
|
20
|
-
# The input source that was compiled.
|
21
|
-
:code,
|
22
|
-
# The command line used for compilation.
|
23
|
-
:command_line
|
24
|
-
) do
|
25
|
-
def to_s
|
26
|
-
<<-EOS
|
27
|
-
> #{command_line}
|
28
|
-
#{stderr}
|
29
|
-
|
30
|
-
[compiling:
|
31
|
-
#{code}
|
32
|
-
]
|
33
|
-
EOS
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
10
|
# Basic interface to compiler frontends.
|
38
11
|
class Compiler
|
39
|
-
# Maximum template recursion depth supported by the compiler.
|
40
|
-
def template_depth
|
41
|
-
raise NotImplementedError
|
42
|
-
end
|
43
|
-
|
44
|
-
# Maximum constexpr recursion depth supported by the compiler.
|
45
|
-
def constexpr_depth
|
46
|
-
raise NotImplementedError
|
47
|
-
end
|
48
|
-
|
49
12
|
# Show the name and the version of the compiler.
|
50
13
|
def to_s
|
51
14
|
raise NotImplementedError
|
52
15
|
end
|
53
16
|
|
54
|
-
#
|
17
|
+
# compile: Path -> Hash
|
55
18
|
#
|
56
19
|
# Compile the given file and return compilation statistics.
|
57
20
|
#
|
58
|
-
# Additional compiler-specific arguments may be specified.
|
59
|
-
#
|
60
|
-
#
|
61
|
-
|
62
|
-
|
63
|
-
def compile_file(file, *args)
|
64
|
-
raise ArgumentError, "invalid filename #{file}" unless File.file? file
|
65
|
-
code = File.read(file)
|
66
|
-
compile_code(code, *args)
|
21
|
+
# Additional compiler-specific arguments may be specified. A
|
22
|
+
# `CompilationError` is raised if the compilation fails for
|
23
|
+
# whatever reason.
|
24
|
+
def compile(file, *args)
|
25
|
+
raise NotImplementedError
|
67
26
|
end
|
68
27
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
tmp.close
|
80
|
-
compile_file(tmp.path, *args)
|
81
|
-
ensure
|
82
|
-
tmp.unlink
|
28
|
+
def self.guess_from_binary(binary)
|
29
|
+
stdout, stderr, status = Open3.capture3("#{binary} --version")
|
30
|
+
case stdout
|
31
|
+
when /\(GCC\)/
|
32
|
+
return GCC.new(binary)
|
33
|
+
when /clang/
|
34
|
+
return Clang.new(binary)
|
35
|
+
else
|
36
|
+
raise ArgumentError("unknown compiler #{binary}")
|
37
|
+
end
|
83
38
|
end
|
84
39
|
end # class Compiler
|
85
40
|
|
@@ -89,41 +44,47 @@ EOS
|
|
89
44
|
raise "#{binary} not found" unless $?.success?
|
90
45
|
end
|
91
46
|
|
92
|
-
def
|
47
|
+
def compile(file, *args)
|
93
48
|
file = Pathname.new(file).expand_path
|
94
49
|
command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
|
95
50
|
stdout, stderr, status = Open3.capture3(command)
|
96
51
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
52
|
+
if status.success?
|
53
|
+
memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
54
|
+
|
55
|
+
section = -> (title) {
|
56
|
+
title.gsub!(' ', '\s')
|
57
|
+
/(
|
58
|
+
===-+===\n
|
59
|
+
.*#{title}.*\n
|
60
|
+
===-+===\n
|
61
|
+
(.|\n)+?(?====)
|
62
|
+
)|(
|
63
|
+
===-+===\n
|
64
|
+
.*#{title}.*\n
|
65
|
+
===-+===\n
|
66
|
+
(.|\n)+
|
67
|
+
)/x
|
68
|
+
}
|
69
|
+
time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
|
70
|
+
.match(/(\d+\.?\d+).+?Total/)[1]
|
71
|
+
return {
|
72
|
+
compilation_time: time,
|
73
|
+
memory_usage: memusg
|
74
|
+
}
|
103
75
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
)
|
114
|
-
|
115
|
-
.*#{title}.*\n
|
116
|
-
===-+===\n
|
117
|
-
(.|\n)+
|
118
|
-
)/x
|
119
|
-
}
|
120
|
-
result.wall_time = stderr.match(section["Miscellaneous Ungrouped Timers"])
|
121
|
-
.to_s.match(/(\d+\.?\d+).+?Total/)[1]
|
122
|
-
return result
|
76
|
+
else
|
77
|
+
err_string = <<-EOS
|
78
|
+
> #{command}
|
79
|
+
#{stderr}
|
80
|
+
|
81
|
+
[compiling:
|
82
|
+
#{file.read}
|
83
|
+
]
|
84
|
+
EOS
|
85
|
+
raise CompilationError.new(err_string)
|
86
|
+
end
|
123
87
|
end
|
124
|
-
|
125
|
-
def template_depth; 256; end
|
126
|
-
def constexpr_depth; 512; end
|
127
88
|
end
|
128
89
|
|
129
90
|
class GCC < Compiler
|
@@ -132,37 +93,28 @@ EOS
|
|
132
93
|
raise "#{binary} not found" unless $?.success?
|
133
94
|
end
|
134
95
|
|
135
|
-
def
|
96
|
+
def compile(file, *args)
|
136
97
|
file = Pathname.new(file).expand_path
|
137
98
|
command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
|
138
99
|
stdout, stderr, status = Open3.capture3(command)
|
139
100
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
148
|
-
result.wall_time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
|
149
|
-
return result
|
150
|
-
end
|
151
|
-
|
152
|
-
def template_depth; 900; end
|
153
|
-
def constexpr_depth; 512; end
|
154
|
-
end
|
155
|
-
|
156
|
-
class Compiler
|
157
|
-
def self.guess_from_binary(binary)
|
158
|
-
stdout, stderr, status = Open3.capture3("#{binary} --version")
|
159
|
-
case stdout
|
160
|
-
when /\(GCC\)/
|
161
|
-
return GCC.new(binary)
|
162
|
-
when /clang/
|
163
|
-
return Clang.new(binary)
|
101
|
+
if status.success?
|
102
|
+
time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
|
103
|
+
memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
|
104
|
+
return {
|
105
|
+
compilation_time: time,
|
106
|
+
memory_usage: memusg
|
107
|
+
}
|
164
108
|
else
|
165
|
-
|
109
|
+
err_string = <<-EOS
|
110
|
+
> #{command}
|
111
|
+
#{stderr}
|
112
|
+
|
113
|
+
[compiling:
|
114
|
+
#{file.read}
|
115
|
+
]
|
116
|
+
EOS
|
117
|
+
raise CompilationError.new(err_string)
|
166
118
|
end
|
167
119
|
end
|
168
120
|
end
|
data/lib/benchcc/plot.rb
CHANGED
@@ -3,42 +3,45 @@ require 'gnuplot'
|
|
3
3
|
|
4
4
|
|
5
5
|
module Benchcc
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
6
|
+
Y_FEATURES = [:memory_usage, :compilation_time, :run_time]
|
7
|
+
|
8
|
+
# How do I make this private?
|
9
|
+
DEFAULT_TWEAK = {
|
10
|
+
memory_usage: -> (plot) {
|
11
|
+
plot.ylabel 'Memory usage'
|
12
|
+
plot.decimal 'locale \'en_US.UTF-8\''
|
13
|
+
plot.format 'y \'%.2e kb\''
|
14
|
+
},
|
15
|
+
|
16
|
+
compilation_time: -> (plot) {
|
17
|
+
plot.ylabel 'Compilation time'
|
18
|
+
plot.format 'y \'%.2g s\''
|
19
|
+
},
|
20
|
+
|
21
|
+
run_time: -> (plot) {
|
22
|
+
plot.ylabel 'Run time'
|
23
|
+
plot.format 'y \'%.2g s\''
|
24
|
+
}
|
25
|
+
}
|
26
|
+
|
27
|
+
def plot(output, titles, inputs, x_feature: :input_size, y_feature: :compilation_time, &tweak)
|
28
|
+
raise ArgumentError if not Benchcc::Y_FEATURES.include?(y_feature)
|
29
|
+
tweak ||= Benchcc::DEFAULT_TWEAK[y_feature]
|
25
30
|
|
26
|
-
def plot_time(output, *inputs)
|
27
31
|
Gnuplot.open do |io|
|
28
32
|
Gnuplot::Plot.new(io) do |plot|
|
29
|
-
plot.ylabel 'Compilation time'
|
30
|
-
plot.format 'y "%.2g s"'
|
31
33
|
plot.term 'png'
|
32
34
|
plot.output output
|
33
|
-
plot.data = inputs.map { |file|
|
35
|
+
plot.data = titles.zip(inputs).map { |title, file|
|
34
36
|
csv = CSV.table(file)
|
35
|
-
Gnuplot::DataSet.new([csv[
|
36
|
-
ds.title =
|
37
|
+
Gnuplot::DataSet.new([csv[x_feature], csv[y_feature]]) { |ds|
|
38
|
+
ds.title = title
|
37
39
|
ds.with = 'lines'
|
38
40
|
}
|
39
41
|
}
|
42
|
+
tweak.call(plot)
|
40
43
|
end
|
41
44
|
end
|
42
45
|
end
|
43
|
-
module_function :
|
46
|
+
module_function :plot
|
44
47
|
end
|
data/lib/benchcc/version.rb
CHANGED
data/spec/benchmark_spec.rb
CHANGED
@@ -1,50 +1,45 @@
|
|
1
1
|
require 'benchcc/benchmark'
|
2
|
+
require 'benchcc/compiler'
|
2
3
|
|
3
4
|
require 'rspec'
|
4
5
|
require 'tempfile'
|
5
|
-
require 'timeout'
|
6
6
|
|
7
7
|
|
8
8
|
describe Benchcc.method(:benchmark) do
|
9
|
-
it('should
|
10
|
-
|
11
|
-
envs = [{time: timeout - 0.1}, {time: timeout + 0.1}, {time: timeout - 0.2}]
|
9
|
+
it('should not throw') {
|
10
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
12
11
|
Tempfile.create('') do |file|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
12
|
+
expect {
|
13
|
+
data = Benchcc.benchmark(file.path, envs) do |file, env|
|
14
|
+
{
|
15
|
+
compilation_time: 'foo',
|
16
|
+
memory_usage: 'bar',
|
17
|
+
run_time: 'baz'
|
18
|
+
}
|
19
|
+
end
|
20
|
+
}.not_to raise_error
|
21
21
|
end
|
22
22
|
}
|
23
23
|
|
24
|
-
it('should
|
25
|
-
envs = [{
|
24
|
+
it('should stop on timeout') {
|
25
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
26
26
|
Tempfile.create('') do |file|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
expect(data).to eq(envs)
|
27
|
+
expect {
|
28
|
+
data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
|
29
|
+
sleep(0.2)
|
30
|
+
end
|
31
|
+
}.not_to raise_error
|
33
32
|
end
|
34
33
|
}
|
35
|
-
end
|
36
34
|
|
37
|
-
|
38
|
-
|
39
|
-
envs = [{x: 1, y:-1}, {x: 2, y:-2}, {x: 3, y:-3}]
|
35
|
+
it('should stop on CompilationError') {
|
36
|
+
envs = [{input_size: 1}, {input_size: 2}]
|
40
37
|
Tempfile.create('') do |file|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
}.to_csv
|
47
|
-
expect(csv).to eq("x,y\n1,-1\n2,-2\n3,-3\n")
|
38
|
+
expect {
|
39
|
+
data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
|
40
|
+
raise Benchcc::CompilationError.new
|
41
|
+
end
|
42
|
+
}.not_to raise_error
|
48
43
|
end
|
49
44
|
}
|
50
45
|
end
|
data/spec/compiler_spec.rb
CHANGED
@@ -6,28 +6,6 @@ require 'tempfile'
|
|
6
6
|
|
7
7
|
|
8
8
|
describe Benchcc::Compiler do
|
9
|
-
describe :compile_file do
|
10
|
-
it {
|
11
|
-
class Mock < Benchcc::Compiler
|
12
|
-
def compile_code(code); code; end
|
13
|
-
end
|
14
|
-
|
15
|
-
file = Tempfile.new('')
|
16
|
-
file.write("whatever")
|
17
|
-
expect(Mock.new.compile_file(file.path)).to eq(file.read)
|
18
|
-
}
|
19
|
-
end
|
20
|
-
|
21
|
-
describe :compile_code do
|
22
|
-
it {
|
23
|
-
class Mock < Benchcc::Compiler
|
24
|
-
def compile_file(file); File.read(file); end
|
25
|
-
end
|
26
|
-
|
27
|
-
expect(Mock.new.compile_code("whatever")).to eq("whatever")
|
28
|
-
}
|
29
|
-
end
|
30
|
-
|
31
9
|
describe :guess_from_binary do
|
32
10
|
it('can guess clang') {
|
33
11
|
expect(
|
@@ -58,17 +36,17 @@ end
|
|
58
36
|
@valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
|
59
37
|
}
|
60
38
|
|
61
|
-
describe :
|
39
|
+
describe :compile do
|
62
40
|
it('fails cleanly on invalid input') {
|
63
41
|
expect {
|
64
|
-
@cc.
|
42
|
+
@cc.compile(@invalid)
|
65
43
|
}.to raise_error(Benchcc::CompilationError)
|
66
44
|
}
|
67
45
|
|
68
46
|
it('returns statistics on valid input') {
|
69
47
|
expect(
|
70
|
-
@cc.
|
71
|
-
).to be_instance_of(
|
48
|
+
@cc.compile(@valid, '-o /dev/null')
|
49
|
+
).to be_instance_of(Hash)
|
72
50
|
}
|
73
51
|
end
|
74
52
|
end
|
data/spec/plot_spec.rb
CHANGED
@@ -15,15 +15,13 @@ describe 'plots' do
|
|
15
15
|
FileUtils.remove_entry(@tmpdir)
|
16
16
|
end
|
17
17
|
|
18
|
-
describe Benchcc.method(:
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
expect { Benchcc.plot_time(@out, @csv) }.not_to raise_error
|
27
|
-
}
|
18
|
+
describe Benchcc.method(:plot) do
|
19
|
+
Benchcc::Y_FEATURES.each do |feature|
|
20
|
+
it {
|
21
|
+
expect {
|
22
|
+
Benchcc.plot(@out, [''], [@csv], y_feature: feature)
|
23
|
+
}.not_to raise_error
|
24
|
+
}
|
25
|
+
end
|
28
26
|
end
|
29
27
|
end
|
data/spec/src/plot.csv
CHANGED
@@ -1,3 +1,3 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
1
|
+
input_size, memory_usage, compilation_time, run_time
|
2
|
+
1, 2, 3, 4
|
3
|
+
5, 6, 7, 8
|