benchcc 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 661158a9d7af70f1063d55e5848506701c1b497e
-  data.tar.gz: 1e2b3c71ecf1c3bc84b214d212ab8033e66428a1
+  metadata.gz: 1639266252ed6664cf61f275325fcb29e864341a
+  data.tar.gz: 3659e30ee50cfec284f7837ea705e778647c8493
 SHA512:
-  metadata.gz: f0f1c6647a9a77860f83908b5bd102f9f468ee3e6fc968625cb33693c6a803c4ec304666f2b81031e535c6df15674e465c7c1a18646ce6d9575667e1b0fedc41
-  data.tar.gz: ee7d8500e0fbbbecf20cb776ef75c7259c5c9d357a4aa35d44311e8799c16513ab40366294ee78f00d10e8e70fe7d475dfa2dc09df1cd95fa58bfe4cffbeeeaa
+  metadata.gz: 41b54dc111a54b67ceed9d434c8e3b3b39ef3074f0883666444a32ace49070a5c04510bac6e51656768836e48168be1f909f1651de3ffce0be22b436eac5310b
+  data.tar.gz: 9c1c1d2557408c9cccf48bfd730dc29fa074d22f1d9e0baeb07c42b15cf60b614a67abf8ec57958ac9b0ef08f7d479c17ee6a8d5193cbace8d0673153d1c40f9

data/lib/benchcc/benchmark.rb

@@ -1,9 +1,9 @@
 require_relative 'compiler'
+
 require 'csv'
 require 'pathname'
 require 'ruby-progressbar'
 require 'tilt'
-require 'timeout'
 
 
 module Benchcc
@@ -20,31 +20,45 @@ module Benchcc
     end
   end
 
-  def benchmark(erb_file, environments, timeout: 10,
-                evaluate_erb_relative_to: File.dirname(erb_file), &bench)
+  def benchmark(
+      erb_file:, # String or Pathname
+      environments:, # Array of Hash
+      compilation_timeout:, # Int
+      execution_timeout:, # Int
+      evaluate_erb_relative_to:, # String or Pathname
+      features:, # Array of Symbol
+      compiler_executable:, # String
+      compiler_id:, # String
+      compiler_options: # Array of String
+    )
     erb_file = Pathname.new(erb_file)
     progress = ProgressBar.create(format: '%p%% | %B |', total: environments.size)
+    compiler = Benchcc::which(compiler_id)
 
     data = CSV.generate({headers: :first_row}) do |csv|
-      csv << [:input_size, :compilation_time, :memory_usage, :run_time]
+      csv << [:input_size] + features
 
       environments.each do |env|
         code = Renderer.new(evaluate_erb_relative_to).render(erb_file, **env)
         begin
+          row = {input_size: env[:input_size]}
           Tempfile.create([erb_file.basename, '.cpp']) do |tmp|
             tmp.write(code) && tmp.close
-            bench_data = Timeout::timeout(timeout) {
-              bench.call(tmp.path, env)
-            }
-            row = {
-              input_size: env[:input_size],
-              compilation_time: bench_data[:compilation_time],
-              memory_usage: bench_data[:memory_usage],
-              run_time: bench_data[:run_time] # TODO: implement this
-            }
-            csv << row
+
+            row.merge!(
+              compiler.call(
+                input_file: tmp.path,
+                features: features,
+                compiler_executable: compiler_executable,
+                compiler_options: compiler_options,
+                compilation_timeout: compilation_timeout,
+                execution_timeout: execution_timeout
+              )
+            )
           end
-        rescue CompilationError, Timeout::Error => e
+
+          csv << row
+        rescue ExecutionError, CompilationError, Timeout::Error => e
           $stderr << e
           break
         end

data/lib/benchcc/compiler.rb

@@ -3,121 +3,108 @@ require_relative 'ext/string'
 require 'open3'
 require 'pathname'
 require 'tempfile'
+require 'timeout'
 
 
 module Benchcc
   class CompilationError < RuntimeError
   end
 
-  # Basic interface to compiler frontends.
-  class Compiler
-    # Show the name and the version of the compiler.
-    def to_s
-      raise NotImplementedError
-    end
-
-    # compile: Path -> Hash
-    #
-    # Compile the given file and return compilation statistics.
-    #
-    # Additional compiler-specific arguments may be specified. A
-    # `CompilationError` is raised if the compilation fails for
-    # whatever reason.
-    def compile(file, *args)
-      raise NotImplementedError
-    end
-
-    def self.guess_from_binary(binary)
-      stdout, stderr, status = Open3.capture3("#{binary} --version")
-      case stdout
-      when /\(GCC\)/
-        return GCC.new(binary)
-      when /clang/
-        return Clang.new(binary)
-      else
-        raise ArgumentError("unknown compiler #{binary}")
-      end
-    end
-  end # class Compiler
-
-  class Clang < Compiler
-    def initialize(binary)
-      @exe = `which #{binary}`.strip
-      raise "#{binary} not found" unless $?.success?
-    end
+  class ExecutionError < RuntimeError
+  end
 
-    def compile(file, *args)
-      file = Pathname.new(file).expand_path
-      command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
-      stdout, stderr, status = Open3.capture3(command)
-
-      if status.success?
-        memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
-
-        section = -> (title) {
-          title.gsub!(' ', '\s')
-          /(
-              ===-+===\n
-              .*#{title}.*\n
-              ===-+===\n
-              (.|\n)+?(?====)
-          )|(
-              ===-+===\n
-              .*#{title}.*\n
-              ===-+===\n
-              (.|\n)+
-          )/x
-        }
-        time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
-                     .match(/(\d+\.?\d+).+?Total/)[1]
-        return {
-          compilation_time: time,
-          memory_usage: memusg
+  class Clang
+    def call(input_file:, features:, compiler_executable:, compiler_options:,
+             compilation_timeout:, execution_timeout:)
+      stats = {}
+      Dir.mktmpdir do |tmp_dir|
+        if features.include?(:compilation_time)
+          compiler_options << '-ftime-report'
+        end
+
+        if features.include?(:execution_time)
+          compiler_options << "-o#{tmp_dir}/a.out"
+        end
+
+        if features.include?(:memory_usage)
+          compiler_executable = "/usr/bin/time -l #{compiler_executable}"
+        end
+
+        command = "#{compiler_executable} #{compiler_options.join(' ')} #{input_file}"
+        stdout, stderr, status = Timeout::timeout(compilation_timeout) {
+          Open3.capture3(command)
         }
 
-      else
-        err_string = <<-EOS.strip_heredoc
-          > #{command}
-          #{stderr}
-
-          [compiling:
-          #{file.read}
-          ]
-        EOS
-        raise CompilationError.new(err_string)
+        if not status.success?
+          raise CompilationError.new(<<-EOS.strip_heredoc
+            > #{command}
+            #{stderr}
+
+            [compiling:
+            #{input_file.read}
+            ]
+          EOS
+          )
+        end
+
+        if features.include?(:compilation_time)
+          section = -> (title) {
+            title.gsub!(' ', '\s')
+            /(
+                ===-+===\n
+                .*#{title}.*\n
+                ===-+===\n
+                (.|\n)+?(?====)
+            )|(
+                ===-+===\n
+                .*#{title}.*\n
+                ===-+===\n
+                (.|\n)+
+            )/x
+          }
+          time = stderr.match(section["Miscellaneous Ungrouped Timers"]).to_s
+                       .match(/(\d+\.?\d+).+?Total/)[1]
+          stats.merge!({compilation_time: time})
+        end
+
+        if features.include?(:memory_usage)
+          memusg = stderr.match(/(\d+)\s+maximum/)[1]
+          stats.merge!({memory_usage: memusg})
+        end
+
+        if features.include?(:execution_time)
+          command = "/usr/bin/time #{tmp_dir}/a.out"
+          stdout, stderr, status = Timeout::timeout(execution_timeout) {
+            Open3.capture3(command)
+          }
+          if status.success?
+            time = stderr.match(/(\d+\.?\d*)\s+real/)[1]
+            stats.merge!({execution_time: time})
+          else
+            raise ExecutionError.new(<<-EOS.strip_heredoc
+              > #{command}
+              #{stderr}
+
+              [running:
+              #{input_file.read}
+              ]
+            EOS
+            )
+          end
+        end
+
+        return stats
       end
     end
   end
 
-  class GCC < Compiler
-    def initialize(binary)
-      @exe = `which #{binary}`.strip
-      raise "#{binary} not found" unless $?.success?
-    end
-
-    def compile(file, *args)
-      file = Pathname.new(file).expand_path
-      command = "/usr/bin/time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
-      stdout, stderr, status = Open3.capture3(command)
-
-      if status.success?
-        time = stderr.match(/TOTAL.+/).to_s.split[-3].to_f
-        memusg = stderr.match(/(\d+)\s+maximum/)[1].to_i
-        return {
-          compilation_time: time,
-          memory_usage: memusg
-        }
-      else
-        err_string = <<-EOS.strip_heredoc
-          > #{command}
-          #{stderr}
-
-          [compiling:
-          #{file.read}
-          ]
-        EOS
-        raise CompilationError.new(err_string)
-      end
+  def which(compiler_id)
+    case compiler_id
+    when 'Clang'
+      return Clang.new
+    else
+      raise ArgumentError.new("Unsupported compiler id: #{compiler_id}")
     end
   end
+  module_function :which
 end

data/lib/benchcc/version.rb

@@ -1,3 +1,3 @@
 module Benchcc
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

data/spec/benchmark_spec.rb

@@ -10,35 +10,17 @@ describe Benchcc.method(:benchmark) do
     envs = [{input_size: 1}, {input_size: 2}]
     Tempfile.create('') do |file|
       expect {
-        data = Benchcc.benchmark(file.path, envs) do |file, env|
-          {
-            compilation_time: 'foo',
-            memory_usage: 'bar',
-            run_time: 'baz'
-          }
-        end
-      }.not_to raise_error
-    end
-  }
-
-  it('should stop on timeout') {
-    envs = [{input_size: 1}, {input_size: 2}]
-    Tempfile.create('') do |file|
-      expect {
-        data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
-          sleep(0.2)
-        end
-      }.not_to raise_error
-    end
-  }
-
-  it('should stop on CompilationError') {
-    envs = [{input_size: 1}, {input_size: 2}]
-    Tempfile.create('') do |file|
-      expect {
-        data = Benchcc.benchmark(file.path, envs, timeout: 0.1) do |file, env|
-          raise Benchcc::CompilationError.new
-        end
+        data = Benchcc.benchmark(
+          erb_file: file.path,
+          environments: [],
+          compilation_timeout: 10,
+          execution_timeout: 10,
+          evaluate_erb_relative_to: '',
+          features: [],
+          compiler_executable: `which clang++`,
+          compiler_id: 'Clang',
+          compiler_options: []
+        )
       }.not_to raise_error
     end
   }

data/spec/compiler_spec.rb

@@ -2,52 +2,54 @@ require 'benchcc/compiler'
 
 require 'pathname'
 require 'rspec'
-require 'tempfile'
 
 
-describe Benchcc::Compiler do
-  describe :guess_from_binary do
-    it('can guess clang') {
-      expect(
-        Benchcc::Compiler.guess_from_binary(`which clang`.strip)
-      ).to be_instance_of(Benchcc::Clang)
+[['Clang', 'clang++']].each do |compiler_id, executable|
+  describe compiler_id do
+    before(:each) {
+      @cc = Benchcc::which(compiler_id)
+      @invalid = Pathname.new('src/invalid.cpp').expand_path(File.dirname(__FILE__))
+      @invalid_runtime = Pathname.new('src/invalid_runtime.cpp').expand_path(File.dirname(__FILE__))
+      @valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
     }
 
-    # GCC is aliased to clang on OSX, so this test fails.
-    # it('can guess gcc') {
-    #   expect(
-    #     Benchcc::Compiler.guess_from_binary(`which gcc`.strip)
-    #   ).to be_instance_of(Benchcc::GCC)
-    # }
-
-    it('fails otherwise') {
+    it('fails cleanly on invalid input') {
       expect {
-        Benchcc::Compiler.guess_from_binary('not_a_compiler')
-      }.to raise_error
+        @cc.call(
+          input_file: @invalid,
+          features: [],
+          compiler_executable: executable,
+          compiler_options: [],
+          compilation_timeout: 10,
+          execution_timeout: 10
+        )
+      }.to raise_error(Benchcc::CompilationError)
     }
-  end
-end
 
-[[Benchcc::Clang, 'clang++'], [Benchcc::GCC, 'g++']].each do |compiler, which|
-  describe compiler do
-    before(:each) {
-      @cc = compiler.new(which)
-      @invalid = Pathname.new('src/invalid.cpp').expand_path(File.dirname(__FILE__))
-      @valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
+    it('fails cleanly on runtime error') {
+      expect {
+        @cc.call(
+          input_file: @invalid_runtime,
+          features: [:execution_time],
+          compiler_executable: executable,
+          compiler_options: [],
+          compilation_timeout: 10,
+          execution_timeout: 10
+        )
+      }.to raise_error(Benchcc::ExecutionError)
     }
 
-    describe :compile do
-      it('fails cleanly on invalid input') {
-        expect {
-          @cc.compile(@invalid)
-        }.to raise_error(Benchcc::CompilationError)
-      }
-
-      it('returns statistics on valid input') {
-        expect(
-          @cc.compile(@valid, '-o /dev/null')
-        ).to be_instance_of(Hash)
-      }
-    end
+    it('returns statistics on valid input') {
+      expect(
+        @cc.call(
+          input_file: @valid,
+          features: [:execution_time, :compilation_time, :memory_usage],
+          compiler_executable: executable,
+          compiler_options: ['-o/dev/null'],
+          compilation_timeout: 10,
+          execution_timeout: 10
+        )
+      ).to be_instance_of(Hash)
+    }
   end
 end

data/spec/src/invalid_runtime.cpp

@@ -0,0 +1,5 @@
+#include <cassert>
+
+int main() {
+    assert(false);
+}

data/spec/src/valid.cpp

@@ -1 +1,7 @@
-int main() {}
+#include <chrono>
+#include <thread>
+
+
+int main() {
+    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchcc
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Louis Dionne
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-20 00:00:00.000000000 Z
+date: 2014-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gnuplot
@@ -115,6 +115,7 @@ files:
 - spec/numeric_spec.rb
 - spec/plot_spec.rb
 - spec/src/invalid.cpp
+- spec/src/invalid_runtime.cpp
 - spec/src/plot.csv
 - spec/src/valid.cpp
 homepage: https://github.com/ldionne/benchcc