benchcc 0.0.7 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e1265188ac7addbc5b64c2333c04a01188169fd1
4
- data.tar.gz: a4c62779f8f2cbe87e18de0dd3bf9ac3f965be3b
3
+ metadata.gz: 6565e1423e649ab808388cd0d7fe2ce78fc8bf38
4
+ data.tar.gz: 8b68dc1cdf7e37e92a76731cf4e3407fcbe11e83
5
5
  SHA512:
6
- metadata.gz: 82fb539a1f350af27c819e0fc6446b7b45a49e4a8e0e3cf5a5ba349ccf60981c97d3c9ffcdd7812b209f757f7f08da458d461f6d86731da136dcedd5e53fc4b9
7
- data.tar.gz: 8c5e23eeb3468af278b08a88d3454123ea96b9f477d211b2ab11fe993c7ccb010abcac750e7837f66406a3c1448a5741e344652278b4a94dce808f0783194385
6
+ metadata.gz: fc94f060eb50fb5cdfd2e325fdfbf21bdb0215ce790b6706a33f34c4872834ab2a9eaaa8fc8e157bdec351d6cc35833d9c4c479c7cb2efcd098df3cb40e449e5
7
+ data.tar.gz: 52b0e9d48f49151e1563859b1709165217e9e4edc12ce194dfcc4719d7885c8843fc4f8502a1f1cdc992d2b34781376bf47331970bcddd2ec87d926af4a2bb75
@@ -20,34 +20,40 @@ module Benchcc
20
20
  end
21
21
  end
22
22
 
23
- def benchmark(file, envs, timeout: 10,
24
- relative_to: File.dirname(file),
25
- stderr: $stderr,
26
- progressbar_format: "#{file} %p%% | %B |",
27
- &block)
28
- progress = ProgressBar.create(format: progressbar_format, total: envs.size)
29
- data = []
30
- envs.each do |env|
31
- code = Renderer.new(relative_to).render(file, **env)
32
- begin
33
- Timeout::timeout(timeout) { data << env.merge(block.call(code).to_h) }
34
- rescue CompilationError, Timeout::Error => e
35
- stderr << e
36
- break
37
- end
38
- progress.increment
39
- end
23
+ def benchmark(erb_file, environments, timeout: 10,
24
+ evaluate_erb_relative_to: File.dirname(erb_file), &bench)
25
+ erb_file = Pathname.new(erb_file)
26
+ progress = ProgressBar.create(format: '%p%% | %B |', total: environments.size)
40
27
 
41
- def data.to_csv
42
- return CSV.generate(headers: :first_row) do |csv|
43
- csv << self.first.keys unless self.empty?
44
- self.each { |line| csv << line.values }
28
+ data = CSV.generate({headers: :first_row}) do |csv|
29
+ csv << [:input_size, :compilation_time, :memory_usage, :run_time]
30
+
31
+ environments.each do |env|
32
+ code = Renderer.new(evaluate_erb_relative_to).render(erb_file, **env)
33
+ begin
34
+ Tempfile.create([erb_file.basename, '.cpp']) do |tmp|
35
+ tmp.write(code) && tmp.close
36
+ bench_data = Timeout::timeout(timeout) {
37
+ bench.call(tmp.path, env)
38
+ }
39
+ row = {
40
+ input_size: env[:input_size],
41
+ compilation_time: bench_data[:compilation_time],
42
+ memory_usage: bench_data[:memory_usage],
43
+ run_time: bench_data[:run_time] # TODO: implement this
44
+ }
45
+ csv << row
46
+ end
47
+ rescue CompilationError, Timeout::Error => e
48
+ $stderr << e
49
+ break
50
+ end
51
+ progress.increment
45
52
  end
46
53
  end
47
-
48
54
  return data
49
55
  ensure
50
- progress.finish
56
+ progress.finish
51
57
  end
52
58
  module_function :benchmark
53
59
  end
@@ -7,79 +7,34 @@ module Benchcc
7
7
  class CompilationError < RuntimeError
8
8
  end
9
9
 
10
- # Structure holding various information about the compilation of a file.
11
- CompilationResult = Struct.new('CompilationResult',
12
- # The time taken to compile.
13
- :wall_time,
14
- # The peak memory usage during compilation.
15
- :peak_memusg,
16
- # The standard error produced during compilation.
17
- :stderr,
18
- # The standard output produced during compilation.
19
- :stdout,
20
- # The input source that was compiled.
21
- :code,
22
- # The command line used for compilation.
23
- :command_line
24
- ) do
25
- def to_s
26
- <<-EOS
27
- > #{command_line}
28
- #{stderr}
29
-
30
- [compiling:
31
- #{code}
32
- ]
33
- EOS
34
- end
35
- end
36
-
37
10
  # Basic interface to compiler frontends.
38
11
  class Compiler
39
- # Maximum template recursion depth supported by the compiler.
40
- def template_depth
41
- raise NotImplementedError
42
- end
43
-
44
- # Maximum constexpr recursion depth supported by the compiler.
45
- def constexpr_depth
46
- raise NotImplementedError
47
- end
48
-
49
12
  # Show the name and the version of the compiler.
50
13
  def to_s
51
14
  raise NotImplementedError
52
15
  end
53
16
 
54
- # compile_file: Path -> CompilationResult
17
+ # compile: Path -> Hash
55
18
  #
56
19
  # Compile the given file and return compilation statistics.
57
20
  #
58
- # Additional compiler-specific arguments may be specified.
59
- #
60
- # A `CompilationError` is be raised if the compilation fails for
61
- # whatever reason. Either this method or `compile_code` must be
62
- # implemented in subclasses.
63
- def compile_file(file, *args)
64
- raise ArgumentError, "invalid filename #{file}" unless File.file? file
65
- code = File.read(file)
66
- compile_code(code, *args)
21
+ # Additional compiler-specific arguments may be specified. A
22
+ # `CompilationError` is raised if the compilation fails for
23
+ # whatever reason.
24
+ def compile(file, *args)
25
+ raise NotImplementedError
67
26
  end
68
27
 
69
- # compile_code: String -> CompilationResult
70
- #
71
- # Compile the given string and return compilation statistics.
72
- #
73
- # This method has the same behavior as `compile_file`, except the code
74
- # is given as-is instead of being in a file. Either this method or
75
- # `compile_file` must be implemented in subclasses.
76
- def compile_code(code, *args)
77
- tmp = Tempfile.new(["", '.cpp'])
78
- tmp.write(code)
79
- tmp.close
80
- compile_file(tmp.path, *args)
81
- ensure
82
- tmp.unlink
28
+ def self.guess_from_binary(binary)
29
+ stdout, stderr, status = Open3.capture3("#{binary} --version")
30
+ case stdout
31
+ when /\(GCC\)/
32
+ return GCC.new(binary)
33
+ when /clang/
34
+ return Clang.new(binary)
35
+ else
36
+ raise ArgumentError("unknown compiler #{binary}")
37
+ end
83
38
  end
84
39
  end # class Compiler
85
40
 
@@ -89,41 +44,47 @@ EOS
89
44
  raise "#{binary} not found" unless $?.success?
90
45
  end
91
46
 
92
- def compile_file(file, *args)
47
+ def compile(file, *args)
93
48
  file = Pathname.new(file).expand_path
94
49
  command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
95
50
  stdout, stderr, status = Open3.capture3(command)
96
51
 
97
- result = CompilationResult.new
98
- result.stderr = stderr
99
- result.stdout = stdout
100
- result.code = file.read
101
- result.command_line = command
102
- raise CompilationError.new(result) unless status.success?
52
+ if status.success?
53
+ memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
54
+
55
+ section = -> (title) {
56
+ title.gsub!(' ', '\s')
57
+ /(
58
+ ===-+===\n
59
+ .*#{title}.*\n
60
+ ===-+===\n
61
+ (.|\n)+?(?====)
62
+ )|(
63
+ ===-+===\n
64
+ .*#{title}.*\n
65
+ ===-+===\n
66
+ (.|\n)+
67
+ )/x
68
+ }
69
+ time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
70
+ .match(/(\d+\.?\d+).+?Total/)[1]
71
+ return {
72
+ compilation_time: time,
73
+ memory_usage: memusg
74
+ }
103
75
 
104
- result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
105
-
106
- section = -> (title) {
107
- title.gsub!(' ', '\s')
108
- /(
109
- ===-+===\n
110
- .*#{title}.*\n
111
- ===-+===\n
112
- (.|\n)+?(?====)
113
- )|(
114
- ===-+===\n
115
- .*#{title}.*\n
116
- ===-+===\n
117
- (.|\n)+
118
- )/x
119
- }
120
- result.wall_time = stderr.match(section["Miscellaneous Ungrouped Timers"])
121
- .to_s.match(/(\d+\.?\d+).+?Total/)[1]
122
- return result
76
+ else
77
+ err_string = <<-EOS
78
+ > #{command}
79
+ #{stderr}
80
+
81
+ [compiling:
82
+ #{file.read}
83
+ ]
84
+ EOS
85
+ raise CompilationError.new(err_string)
86
+ end
123
87
  end
124
-
125
- def template_depth; 256; end
126
- def constexpr_depth; 512; end
127
88
  end
128
89
 
129
90
  class GCC < Compiler
@@ -132,37 +93,28 @@ EOS
132
93
  raise "#{binary} not found" unless $?.success?
133
94
  end
134
95
 
135
- def compile_file(file, *args)
96
+ def compile(file, *args)
136
97
  file = Pathname.new(file).expand_path
137
98
  command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
138
99
  stdout, stderr, status = Open3.capture3(command)
139
100
 
140
- result = CompilationResult.new
141
- result.stderr = stderr
142
- result.stdout = stdout
143
- result.code = file.read
144
- result.command_line = command
145
- raise CompilationError.new(result) unless status.success?
146
-
147
- result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
148
- result.wall_time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
149
- return result
150
- end
151
-
152
- def template_depth; 900; end
153
- def constexpr_depth; 512; end
154
- end
155
-
156
- class Compiler
157
- def self.guess_from_binary(binary)
158
- stdout, stderr, status = Open3.capture3("#{binary} --version")
159
- case stdout
160
- when /\(GCC\)/
161
- return GCC.new(binary)
162
- when /clang/
163
- return Clang.new(binary)
101
+ if status.success?
102
+ time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
103
+ memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
104
+ return {
105
+ compilation_time: time,
106
+ memory_usage: memusg
107
+ }
164
108
  else
165
- raise ArgumentError("unknown compiler #{binary}")
109
+ err_string = <<-EOS
110
+ > #{command}
111
+ #{stderr}
112
+
113
+ [compiling:
114
+ #{file.read}
115
+ ]
116
+ EOS
117
+ raise CompilationError.new(err_string)
166
118
  end
167
119
  end
168
120
  end
data/lib/benchcc/plot.rb CHANGED
@@ -3,42 +3,45 @@ require 'gnuplot'
3
3
 
4
4
 
5
5
  module Benchcc
6
- def plot_memusg(output, *inputs)
7
- Gnuplot.open do |io|
8
- Gnuplot::Plot.new(io) do |plot|
9
- plot.ylabel 'Memory usage'
10
- plot.decimal "locale 'en_US.UTF-8'"
11
- plot.format 'y "%.2e kb"'
12
- plot.term 'png'
13
- plot.output output
14
- plot.data = inputs.map { |file|
15
- csv = CSV.table(file)
16
- Gnuplot::DataSet.new([csv[:x], csv[:peak_memusg]]) { |ds|
17
- ds.title = file
18
- ds.with = 'lines'
19
- }
20
- }
21
- end
22
- end
23
- end
24
- module_function :plot_memusg
6
+ Y_FEATURES = [:memory_usage, :compilation_time, :run_time]
7
+
8
+ # How do I make this private?
9
+ DEFAULT_TWEAK = {
10
+ memory_usage: -> (plot) {
11
+ plot.ylabel 'Memory usage'
12
+ plot.decimal 'locale \'en_US.UTF-8\''
13
+ plot.format 'y \'%.2e kb\''
14
+ },
15
+
16
+ compilation_time: -> (plot) {
17
+ plot.ylabel 'Compilation time'
18
+ plot.format 'y \'%.2g s\''
19
+ },
20
+
21
+ run_time: -> (plot) {
22
+ plot.ylabel 'Run time'
23
+ plot.format 'y \'%.2g s\''
24
+ }
25
+ }
26
+
27
+ def plot(output, titles, inputs, x_feature: :input_size, y_feature: :compilation_time, &tweak)
28
+ raise ArgumentError if not Benchcc::Y_FEATURES.include?(y_feature)
29
+ tweak ||= Benchcc::DEFAULT_TWEAK[y_feature]
25
30
 
26
- def plot_time(output, *inputs)
27
31
  Gnuplot.open do |io|
28
32
  Gnuplot::Plot.new(io) do |plot|
29
- plot.ylabel 'Compilation time'
30
- plot.format 'y "%.2g s"'
31
33
  plot.term 'png'
32
34
  plot.output output
33
- plot.data = inputs.map { |file|
35
+ plot.data = titles.zip(inputs).map { |title, file|
34
36
  csv = CSV.table(file)
35
- Gnuplot::DataSet.new([csv[:x], csv[:wall_time]]) { |ds|
36
- ds.title = file
37
+ Gnuplot::DataSet.new([csv[x_feature], csv[y_feature]]) { |ds|
38
+ ds.title = title
37
39
  ds.with = 'lines'
38
40
  }
39
41
  }
42
+ tweak.call(plot)
40
43
  end
41
44
  end
42
45
  end
43
- module_function :plot_time
46
+ module_function :plot
44
47
  end
@@ -1,3 +1,3 @@
1
1
  module Benchcc
2
- VERSION = "0.0.7"
2
+ VERSION = "0.0.8"
3
3
  end
@@ -1,50 +1,45 @@
1
1
  require 'benchcc/benchmark'
2
+ require 'benchcc/compiler'
2
3
 
3
4
  require 'rspec'
4
5
  require 'tempfile'
5
- require 'timeout'
6
6
 
7
7
 
8
8
  describe Benchcc.method(:benchmark) do
9
- it('should stop on timeout') {
10
- timeout = 0.3
11
- envs = [{time: timeout - 0.1}, {time: timeout + 0.1}, {time: timeout - 0.2}]
9
+ it('should not throw') {
10
+ envs = [{input_size: 1}, {input_size: 2}]
12
11
  Tempfile.create('') do |file|
13
- file.write('<%= time %>') && file.flush
14
- devnull = File.open(File::NULL, 'w')
15
-
16
- data = Benchcc.benchmark(file.path, envs, timeout: timeout, stderr: devnull) do |code|
17
- sleep(code.to_f)
18
- {time: code.to_f}
19
- end
20
- expect(data).to eq(envs.take_while { |env| env[:time] < timeout })
12
+ expect {
13
+ data = Benchcc.benchmark(file.path, envs) do |file, env|
14
+ {
15
+ compilation_time: 'foo',
16
+ memory_usage: 'bar',
17
+ run_time: 'baz'
18
+ }
19
+ end
20
+ }.not_to raise_error
21
21
  end
22
22
  }
23
23
 
24
- it('should accumulate whatever is returned from the block') {
25
- envs = [{x: 0}, {x: 1}, {x: 2}, {x: 3}]
24
+ it('should stop on timeout') {
25
+ envs = [{input_size: 1}, {input_size: 2}]
26
26
  Tempfile.create('') do |file|
27
- file.write('<%= x %>')
28
- file.flush
29
- data = Benchcc.benchmark(file.path, envs) do |x|
30
- {x: x.to_i}
31
- end
32
- expect(data).to eq(envs)
27
+ expect {
28
+ data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
29
+ sleep(0.2)
30
+ end
31
+ }.not_to raise_error
33
32
  end
34
33
  }
35
- end
36
34
 
37
- describe 'Benchcc.benchmark(...).to_csv' do
38
- it('should output to csv correctly') {
39
- envs = [{x: 1, y:-1}, {x: 2, y:-2}, {x: 3, y:-3}]
35
+ it('should stop on CompilationError') {
36
+ envs = [{input_size: 1}, {input_size: 2}]
40
37
  Tempfile.create('') do |file|
41
- file.write('<%= x %> <%= y %>')
42
- file.flush
43
- csv = Benchcc.benchmark(file.path, envs) { |str|
44
- x, y = str.split(' ')
45
- {x: x.to_i, y: y.to_i}
46
- }.to_csv
47
- expect(csv).to eq("x,y\n1,-1\n2,-2\n3,-3\n")
38
+ expect {
39
+ data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
40
+ raise Benchcc::CompilationError.new
41
+ end
42
+ }.not_to raise_error
48
43
  end
49
44
  }
50
45
  end
@@ -6,28 +6,6 @@ require 'tempfile'
6
6
 
7
7
 
8
8
  describe Benchcc::Compiler do
9
- describe :compile_file do
10
- it {
11
- class Mock < Benchcc::Compiler
12
- def compile_code(code); code; end
13
- end
14
-
15
- file = Tempfile.new('')
16
- file.write("whatever")
17
- expect(Mock.new.compile_file(file.path)).to eq(file.read)
18
- }
19
- end
20
-
21
- describe :compile_code do
22
- it {
23
- class Mock < Benchcc::Compiler
24
- def compile_file(file); File.read(file); end
25
- end
26
-
27
- expect(Mock.new.compile_code("whatever")).to eq("whatever")
28
- }
29
- end
30
-
31
9
  describe :guess_from_binary do
32
10
  it('can guess clang') {
33
11
  expect(
@@ -58,17 +36,17 @@ end
58
36
  @valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
59
37
  }
60
38
 
61
- describe :compile_file do
39
+ describe :compile do
62
40
  it('fails cleanly on invalid input') {
63
41
  expect {
64
- @cc.compile_file(@invalid)
42
+ @cc.compile(@invalid)
65
43
  }.to raise_error(Benchcc::CompilationError)
66
44
  }
67
45
 
68
46
  it('returns statistics on valid input') {
69
47
  expect(
70
- @cc.compile_file(@valid, '-o /dev/null')
71
- ).to be_instance_of(Benchcc::CompilationResult)
48
+ @cc.compile(@valid, '-o /dev/null')
49
+ ).to be_instance_of(Hash)
72
50
  }
73
51
  end
74
52
  end
data/spec/plot_spec.rb CHANGED
@@ -15,15 +15,13 @@ describe 'plots' do
15
15
  FileUtils.remove_entry(@tmpdir)
16
16
  end
17
17
 
18
- describe Benchcc.method(:plot_memusg) do
19
- it {
20
- expect { Benchcc.plot_memusg(@out, @csv) }.not_to raise_error
21
- }
22
- end
23
-
24
- describe Benchcc.method(:plot_time) do
25
- it {
26
- expect { Benchcc.plot_time(@out, @csv) }.not_to raise_error
27
- }
18
+ describe Benchcc.method(:plot) do
19
+ Benchcc::Y_FEATURES.each do |feature|
20
+ it {
21
+ expect {
22
+ Benchcc.plot(@out, [''], [@csv], y_feature: feature)
23
+ }.not_to raise_error
24
+ }
25
+ end
28
26
  end
29
27
  end
data/spec/src/plot.csv CHANGED
@@ -1,3 +1,3 @@
1
- x, peak_memusg, wall_time
2
- 0, 1, , 2
3
- 1, 2, , 3
1
+ input_size, memory_usage, compilation_time, run_time
2
+ 1, 2, 3, 4
3
+ 5, 6, 7, 8
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: benchcc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.0.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Louis Dionne