benchcc 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1265188ac7addbc5b64c2333c04a01188169fd1
-  data.tar.gz: a4c62779f8f2cbe87e18de0dd3bf9ac3f965be3b
+  metadata.gz: 6565e1423e649ab808388cd0d7fe2ce78fc8bf38
+  data.tar.gz: 8b68dc1cdf7e37e92a76731cf4e3407fcbe11e83
 SHA512:
-  metadata.gz: 82fb539a1f350af27c819e0fc6446b7b45a49e4a8e0e3cf5a5ba349ccf60981c97d3c9ffcdd7812b209f757f7f08da458d461f6d86731da136dcedd5e53fc4b9
-  data.tar.gz: 8c5e23eeb3468af278b08a88d3454123ea96b9f477d211b2ab11fe993c7ccb010abcac750e7837f66406a3c1448a5741e344652278b4a94dce808f0783194385
+  metadata.gz: fc94f060eb50fb5cdfd2e325fdfbf21bdb0215ce790b6706a33f34c4872834ab2a9eaaa8fc8e157bdec351d6cc35833d9c4c479c7cb2efcd098df3cb40e449e5
+  data.tar.gz: 52b0e9d48f49151e1563859b1709165217e9e4edc12ce194dfcc4719d7885c8843fc4f8502a1f1cdc992d2b34781376bf47331970bcddd2ec87d926af4a2bb75

data/lib/benchcc/benchmark.rb

@@ -20,34 +20,40 @@ module Benchcc
     end
   end
 
-  def benchmark(file, envs, timeout: 10,
-                            relative_to: File.dirname(file),
-                            stderr: $stderr,
-                            progressbar_format: "#{file} %p%% | %B |",
-                            &block)
-    progress = ProgressBar.create(format: progressbar_format, total: envs.size)
-    data = []
-    envs.each do |env|
-      code = Renderer.new(relative_to).render(file, **env)
-      begin
-        Timeout::timeout(timeout) { data << env.merge(block.call(code).to_h) }
-      rescue CompilationError, Timeout::Error => e
-        stderr << e
-        break
-      end
-      progress.increment
-    end
+  def benchmark(erb_file, environments, timeout: 10,
+                evaluate_erb_relative_to: File.dirname(erb_file), &bench)
+    erb_file = Pathname.new(erb_file)
+    progress = ProgressBar.create(format: '%p%% | %B |', total: environments.size)
 
-    def data.to_csv
-      return CSV.generate(headers: :first_row) do |csv|
-        csv << self.first.keys unless self.empty?
-        self.each { |line| csv << line.values }
+    data = CSV.generate({headers: :first_row}) do |csv|
+      csv << [:input_size, :compilation_time, :memory_usage, :run_time]
+
+      environments.each do |env|
+        code = Renderer.new(evaluate_erb_relative_to).render(erb_file, **env)
+        begin
+          Tempfile.create([erb_file.basename, '.cpp']) do |tmp|
+            tmp.write(code) && tmp.close
+            bench_data = Timeout::timeout(timeout) {
+              bench.call(tmp.path, env)
+            }
+            row = {
+              input_size: env[:input_size],
+              compilation_time: bench_data[:compilation_time],
+              memory_usage: bench_data[:memory_usage],
+              run_time: bench_data[:run_time] # TODO: implement this
+            }
+            csv << row
+          end
+        rescue CompilationError, Timeout::Error => e
+          $stderr << e
+          break
+        end
+        progress.increment
       end
     end
-
     return data
   ensure
-      progress.finish
+    progress.finish
   end
   module_function :benchmark
 end

data/lib/benchcc/compiler.rb

@@ -7,79 +7,34 @@ module Benchcc
   class CompilationError < RuntimeError
   end
 
-  # Structure holding various information about the compilation of a file.
-  CompilationResult = Struct.new('CompilationResult',
-    # The time taken to compile.
-    :wall_time,
-    # The peak memory usage during compilation.
-    :peak_memusg,
-    # The standard error produced during compilation.
-    :stderr,
-    # The standard output produced during compilation.
-    :stdout,
-    # The input source that was compiled.
-    :code,
-    # The command line used for compilation.
-    :command_line
-  ) do
-    def to_s
-      <<-EOS
-> #{command_line}
-#{stderr}
-
-[compiling:
-#{code}
-]
-EOS
-    end
-  end
-
   # Basic interface to compiler frontends.
   class Compiler
-    # Maximum template recursion depth supported by the compiler.
-    def template_depth
-      raise NotImplementedError
-    end
-
-    # Maximum constexpr recursion depth supported by the compiler.
-    def constexpr_depth
-      raise NotImplementedError
-    end
-
     # Show the name and the version of the compiler.
     def to_s
       raise NotImplementedError
     end
 
-    # compile_file: Path -> CompilationResult
+    # compile: Path -> Hash
     #
     # Compile the given file and return compilation statistics.
     #
-    # Additional compiler-specific arguments may be specified.
-    #
-    # A `CompilationError` is be raised if the compilation fails for
-    # whatever reason. Either this method or `compile_code` must be
-    # implemented in subclasses.
-    def compile_file(file, *args)
-      raise ArgumentError, "invalid filename #{file}" unless File.file? file
-      code = File.read(file)
-      compile_code(code, *args)
+    # Additional compiler-specific arguments may be specified. A
+    # `CompilationError` is raised if the compilation fails for
+    # whatever reason.
+    def compile(file, *args)
+      raise NotImplementedError
     end
 
-    # compile_code: String -> CompilationResult
-    #
-    # Compile the given string and return compilation statistics.
-    #
-    # This method has the same behavior as `compile_file`, except the code
-    # is given as-is instead of being in a file. Either this method or
-    # `compile_file` must be implemented in subclasses.
-    def compile_code(code, *args)
-      tmp = Tempfile.new(["", '.cpp'])
-      tmp.write(code)
-      tmp.close
-      compile_file(tmp.path, *args)
-    ensure
-      tmp.unlink
+    def self.guess_from_binary(binary)
+      stdout, stderr, status = Open3.capture3("#{binary} --version")
+      case stdout
+      when /\(GCC\)/
+        return GCC.new(binary)
+      when /clang/
+        return Clang.new(binary)
+      else
+        raise ArgumentError("unknown compiler #{binary}")
+      end
     end
   end # class Compiler
 
@@ -89,41 +44,47 @@ EOS
       raise "#{binary} not found" unless $?.success?
     end
 
-    def compile_file(file, *args)
+    def compile(file, *args)
       file = Pathname.new(file).expand_path
       command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
       stdout, stderr, status = Open3.capture3(command)
 
-      result = CompilationResult.new
-      result.stderr = stderr
-      result.stdout = stdout
-      result.code = file.read
-      result.command_line = command
-      raise CompilationError.new(result) unless status.success?
+      if status.success?
+        memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
+
+        section = -> (title) {
+          title.gsub!(' ', '\s')
+          /(
+              ===-+===\n
+              .*#{title}.*\n
+              ===-+===\n
+              (.|\n)+?(?====)
+          )|(
+              ===-+===\n
+              .*#{title}.*\n
+              ===-+===\n
+              (.|\n)+
+          )/x
+        }
+        time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
+                     .match(/(\d+\.?\d+).+?Total/)[1]
+        return {
+          compilation_time: time,
+          memory_usage: memusg
+        }
 
-      result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
-
-      section = -> (title) {
-        title.gsub!(' ', '\s')
-        /(
-            ===-+===\n
-            .*#{title}.*\n
-            ===-+===\n
-            (.|\n)+?(?====)
-        )|(
-            ===-+===\n
-            .*#{title}.*\n
-            ===-+===\n
-            (.|\n)+
-        )/x
-      }
-      result.wall_time = stderr.match(section["Miscellaneous Ungrouped Timers"])
-                               .to_s.match(/(\d+\.?\d+).+?Total/)[1]
-      return result
+      else
+        err_string = <<-EOS
+          > #{command}
+          #{stderr}
+
+          [compiling:
+          #{file.read}
+          ]
+        EOS
+        raise CompilationError.new(err_string)
+      end
     end
-
-    def template_depth;  256; end
-    def constexpr_depth; 512; end
   end
 
   class GCC < Compiler
@@ -132,37 +93,28 @@ EOS
       raise "#{binary} not found" unless $?.success?
     end
 
-    def compile_file(file, *args)
+    def compile(file, *args)
       file = Pathname.new(file).expand_path
       command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
       stdout, stderr, status = Open3.capture3(command)
 
-      result = CompilationResult.new
-      result.stderr = stderr
-      result.stdout = stdout
-      result.code = file.read
-      result.command_line = command
-      raise CompilationError.new(result) unless status.success?
-
-      result.peak_memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
-      result.wall_time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
-      return result
-    end
-
-    def template_depth;  900; end
-    def constexpr_depth; 512; end
-  end
-
-  class Compiler
-    def self.guess_from_binary(binary)
-      stdout, stderr, status = Open3.capture3("#{binary} --version")
-      case stdout
-      when /\(GCC\)/
-        return GCC.new(binary)
-      when /clang/
-        return Clang.new(binary)
+      if status.success?
+        time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
+        memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
+        return {
+          compilation_time: time,
+          memory_usage: memusg
+        }
       else
-        raise ArgumentError("unknown compiler #{binary}")
+        err_string = <<-EOS
+          > #{command}
+          #{stderr}
+
+          [compiling:
+          #{file.read}
+          ]
+        EOS
+        raise CompilationError.new(err_string)
       end
     end
   end

data/lib/benchcc/plot.rb

@@ -3,42 +3,45 @@ require 'gnuplot'
 
 
 module Benchcc
-  def plot_memusg(output, *inputs)
-    Gnuplot.open do |io|
-      Gnuplot::Plot.new(io) do |plot|
-        plot.ylabel   'Memory usage'
-        plot.decimal  "locale 'en_US.UTF-8'"
-        plot.format   'y "%.2e kb"'
-        plot.term     'png'
-        plot.output   output
-        plot.data = inputs.map { |file|
-          csv = CSV.table(file)
-          Gnuplot::DataSet.new([csv[:x], csv[:peak_memusg]]) { |ds|
-            ds.title = file
-            ds.with = 'lines'
-          }
-        }
-      end
-    end
-  end
-  module_function :plot_memusg
+  Y_FEATURES = [:memory_usage, :compilation_time, :run_time]
+
+  # How do I make this private?
+  DEFAULT_TWEAK = {
+    memory_usage: -> (plot) {
+      plot.ylabel   'Memory usage'
+      plot.decimal  'locale \'en_US.UTF-8\''
+      plot.format   'y \'%.2e kb\''
+    },
+
+    compilation_time: -> (plot) {
+      plot.ylabel   'Compilation time'
+      plot.format   'y \'%.2g s\''
+    },
+
+    run_time: -> (plot) {
+      plot.ylabel   'Run time'
+      plot.format   'y \'%.2g s\''
+    }
+  }
+
+  def plot(output, titles, inputs, x_feature: :input_size, y_feature: :compilation_time, &tweak)
+    raise ArgumentError if not Benchcc::Y_FEATURES.include?(y_feature)
+    tweak ||= Benchcc::DEFAULT_TWEAK[y_feature]
 
-  def plot_time(output, *inputs)
     Gnuplot.open do |io|
       Gnuplot::Plot.new(io) do |plot|
-        plot.ylabel   'Compilation time'
-        plot.format   'y "%.2g s"'
         plot.term     'png'
         plot.output   output
-        plot.data = inputs.map { |file|
+        plot.data = titles.zip(inputs).map { |title, file|
           csv = CSV.table(file)
-          Gnuplot::DataSet.new([csv[:x], csv[:wall_time]]) { |ds|
-            ds.title = file
+          Gnuplot::DataSet.new([csv[x_feature], csv[y_feature]]) { |ds|
+            ds.title = title
             ds.with = 'lines'
           }
         }
+        tweak.call(plot)
       end
     end
   end
-  module_function :plot_time
+  module_function :plot
 end

data/lib/benchcc/version.rb

@@ -1,3 +1,3 @@
 module Benchcc
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/spec/benchmark_spec.rb

@@ -1,50 +1,45 @@
 require 'benchcc/benchmark'
+require 'benchcc/compiler'
 
 require 'rspec'
 require 'tempfile'
-require 'timeout'
 
 
 describe Benchcc.method(:benchmark) do
-  it('should stop on timeout') {
-    timeout = 0.3
-    envs = [{time: timeout - 0.1}, {time: timeout + 0.1}, {time: timeout - 0.2}]
+  it('should not throw') {
+    envs = [{input_size: 1}, {input_size: 2}]
     Tempfile.create('') do |file|
-      file.write('<%= time %>') && file.flush
-      devnull = File.open(File::NULL, 'w')
-
-      data = Benchcc.benchmark(file.path, envs, timeout: timeout, stderr: devnull) do |code|
-        sleep(code.to_f)
-        {time: code.to_f}
-      end
-      expect(data).to eq(envs.take_while { |env| env[:time] < timeout })
+      expect {
+        data = Benchcc.benchmark(file.path, envs) do |file, env|
+          {
+            compilation_time: 'foo',
+            memory_usage: 'bar',
+            run_time: 'baz'
+          }
+        end
+      }.not_to raise_error
     end
   }
 
-  it('should accumulate whatever is returned from the block') {
-    envs = [{x: 0}, {x: 1}, {x: 2}, {x: 3}]
+  it('should stop on timeout') {
+    envs = [{input_size: 1}, {input_size: 2}]
     Tempfile.create('') do |file|
-      file.write('<%= x %>')
-      file.flush
-      data = Benchcc.benchmark(file.path, envs) do |x|
-        {x: x.to_i}
-      end
-      expect(data).to eq(envs)
+      expect {
+        data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
+          sleep(0.2)
+        end
+      }.not_to raise_error
     end
   }
-end
 
-describe 'Benchcc.benchmark(...).to_csv' do
-  it('should output to csv correctly') {
-    envs = [{x: 1, y:-1}, {x: 2, y:-2}, {x: 3, y:-3}]
+  it('should stop on CompilationError') {
+    envs = [{input_size: 1}, {input_size: 2}]
     Tempfile.create('') do |file|
-      file.write('<%= x %> <%= y %>')
-      file.flush
-      csv = Benchcc.benchmark(file.path, envs) { |str|
-        x, y = str.split(' ')
-        {x: x.to_i, y: y.to_i}
-      }.to_csv
-      expect(csv).to eq("x,y\n1,-1\n2,-2\n3,-3\n")
+      expect {
+        data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
+          raise Benchcc::CompilationError.new
+        end
+      }.not_to raise_error
     end
   }
 end

data/spec/compiler_spec.rb

@@ -6,28 +6,6 @@ require 'tempfile'
 
 
 describe Benchcc::Compiler do
-  describe :compile_file do
-    it {
-      class Mock < Benchcc::Compiler
-        def compile_code(code); code; end
-      end
-
-      file = Tempfile.new('')
-      file.write("whatever")
-      expect(Mock.new.compile_file(file.path)).to eq(file.read)
-    }
-  end
-
-  describe :compile_code do
-    it {
-      class Mock < Benchcc::Compiler
-        def compile_file(file); File.read(file); end
-      end
-
-      expect(Mock.new.compile_code("whatever")).to eq("whatever")
-    }
-  end
-
   describe :guess_from_binary do
     it('can guess clang') {
       expect(
@@ -58,17 +36,17 @@ end
       @valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
     }
 
-    describe :compile_file do
+    describe :compile do
       it('fails cleanly on invalid input') {
         expect {
-          @cc.compile_file(@invalid)
+          @cc.compile(@invalid)
         }.to raise_error(Benchcc::CompilationError)
       }
 
       it('returns statistics on valid input') {
         expect(
-          @cc.compile_file(@valid, '-o /dev/null')
-        ).to be_instance_of(Benchcc::CompilationResult)
+          @cc.compile(@valid, '-o /dev/null')
+        ).to be_instance_of(Hash)
       }
     end
   end

data/spec/plot_spec.rb

@@ -15,15 +15,13 @@ describe 'plots' do
     FileUtils.remove_entry(@tmpdir)
   end
 
-  describe Benchcc.method(:plot_memusg) do
-    it {
-      expect { Benchcc.plot_memusg(@out, @csv) }.not_to raise_error
-    }
-  end
-
-  describe Benchcc.method(:plot_time) do
-    it {
-      expect { Benchcc.plot_time(@out, @csv) }.not_to raise_error
-    }
+  describe Benchcc.method(:plot) do
+    Benchcc::Y_FEATURES.each do |feature|
+      it {
+        expect {
+          Benchcc.plot(@out, [''], [@csv], y_feature: feature)
+        }.not_to raise_error
+      }
+    end
   end
 end

data/spec/src/plot.csv

@@ -1,3 +1,3 @@
-x, peak_memusg, wall_time
-0, 1,         , 2
-1, 2,         , 3
+input_size, memory_usage, compilation_time, run_time
+1, 2, 3, 4
+5, 6, 7, 8

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: benchcc
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Louis Dionne