benchcc 0.0.1

data/benchmarks/type_foldl/mpl11_list.erb.cpp

@@ -0,0 +1,8 @@
+#define BOOST_MPL11_NO_ASSERTIONS
+#include <boost/mpl11/list.hpp>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    list = "boost::mpl11::list<#{xs.join(', ')}>"
+    "boost::mpl11::foldl<#{f}, #{state}, #{list}>::type"
+end %>

data/benchmarks/type_foldl/mpl_list.erb.cpp

@@ -0,0 +1,7 @@
+#include <boost/mpl/fold.hpp>
+<%= Benchcc::MPL::List.new(0...depth).includes %>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    "boost::mpl::fold<#{Benchcc::MPL::List.new(xs)}, #{state}, #{f}>::type"
+end %>

data/benchmarks/type_foldl/mpl_vector.erb.cpp

@@ -0,0 +1,7 @@
+#include <boost/mpl/fold.hpp>
+<%= Benchcc::MPL::Vector.new(0...depth).includes %>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    "boost::mpl::fold<#{Benchcc::MPL::Vector.new(xs)}, #{state}, #{f}>::type"
+end %>

data/benchmarks/value_foldl/_env.rb

@@ -0,0 +1,3 @@
+(0..1000).step(5).map { |depth|
+  { breadth: 1, depth: depth }
+}

data/benchmarks/value_foldl/_main.erb

@@ -0,0 +1,17 @@
+template <typename ...>
+struct result { };
+
+constexpr struct {
+    template <typename State, typename X>
+    constexpr result<State, X> operator()(State, X) const
+    { return {}; }
+} f{};
+
+<%= (0...breadth).map { |breadth|
+    xs = (0...depth).map { |depth| "x#{breadth}<#{depth}>" }
+    <<-EOS
+    constexpr struct { } state#{breadth}{};
+    template <int> struct x#{breadth} { };
+    static const auto go#{breadth} = #{yield 'f', "state#{breadth}", xs};
+    EOS
+}.join("\n") %>

data/benchmarks/value_foldl/fusion_cons.erb.cpp

@@ -0,0 +1,8 @@
+#include <boost/fusion/algorithm/iteration/fold.hpp>
+<%= Benchcc::Fusion::Cons.new(0...depth).includes %>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    cons = Benchcc::Fusion::Cons.new(xs)
+    "boost::fusion::fold(#{cons}{}, #{state}, #{f})"
+end %>

data/benchmarks/value_foldl/fusion_list.erb.cpp

@@ -0,0 +1,9 @@
+<%= Benchcc::Fusion::List.new(0...depth).includes %>
+
+#include <boost/fusion/algorithm/iteration/fold.hpp>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    list = Benchcc::Fusion::List.new(xs)
+    "boost::fusion::fold(#{list}{}, #{state}, #{f})"
+end %>

data/benchmarks/value_foldl/fusion_vector.erb.cpp

@@ -0,0 +1,9 @@
+<%= Benchcc::Fusion::Vector.new(0...depth).includes %>
+
+#include <boost/fusion/algorithm/iteration/fold.hpp>
+
+
+<%= render('_main.erb') do |f, state, xs|
+    vector = Benchcc::Fusion::Vector.new(xs)
+    "boost::fusion::fold(#{vector}{}, #{state}, #{f})"
+end %>

data/benchmarks/value_foldl/hana_list.erb.cpp

@@ -0,0 +1,6 @@
+#include <boost/hana/list.hpp>
+
+<%= render('_main.erb') do |f, state, xs|
+    xs = xs.map { |x| "#{x}{}" }.join(', ')
+    "boost::hana::foldl(#{f}, #{state}, boost::hana::list(#{xs}))"
+end %>

data/lib/benchcc/benchmark.rb

@@ -0,0 +1,53 @@
+require_relative 'compiler'
+require 'csv'
+require 'pathname'
+require 'ruby-progressbar'
+require 'tilt'
+require 'timeout'
+
+
+module Benchcc
+  class Renderer
+    def initialize(relative_to)
+      @relative_to = Pathname.new(relative_to)
+      @locals = {}
+    end
+
+    def render(file, **locals, &block)
+      @locals.merge!(locals)
+      file = Pathname.new(file).expand_path(@relative_to)
+      Tilt::ERBTemplate.new(file).render(self, **@locals, &block)
+    end
+  end
+
+  def benchmark(file, envs, timeout: 10, relative_to: File.dirname(file), &block)
+    progress = ProgressBar.create(format: "#{file} %p%% | %B |",
+                                  total: envs.size)
+    consecutive_errors, data = 0, []
+    envs.each do |env|
+      break if consecutive_errors >= 2
+      code = Renderer.new(relative_to).render(file, **env)
+      begin
+        Timeout::timeout(timeout) { data << env.merge(block.call(code)) }
+        consecutive_errors = 0
+      rescue CompilationError, Timeout::Error => e
+        $stderr << e
+        consecutive_errors += 1
+      end
+      progress.increment
+    end
+    return data
+  ensure
+      progress.finish
+  end
+  module_function :benchmark
+
+  def benchmark_to_csv(file, envs, out, timeout: 10, relative_to: File.dirname(file), &block)
+    data = benchmark(file, envs, timeout: timeout, relative_to: relative_to, &block)
+    CSV.open(out, 'wb') do |csv|
+      csv << data.first.keys unless data.empty?
+      data.each { |line| csv << line.values }
+    end
+  end
+  module_function :benchmark_to_csv
+end

data/lib/benchcc/compiler.rb

@@ -0,0 +1,134 @@
+require 'open3'
+require 'pathname'
+require 'tempfile'
+
+
+module Benchcc
+  class CompilationError < RuntimeError
+    def initialize(command_line, code, compiler_error_message)
+      @cli = command_line
+      @code = code
+      @compiler_stderr = compiler_error_message
+    end
+
+    def to_s
+      <<-EOS
+  compilation failed when invoking "#{@cli}"
+  compiler error message was:
+  #{'-' * 80}
+  #{@compiler_stderr}
+
+  full compiled file was:
+  #{'-' * 80}
+  #{@code}
+  EOS
+    end
+  end
+
+  # Basic interface to compiler frontends.
+  class Compiler
+    # Maximum template recursion depth supported by the compiler.
+    def template_depth
+      raise NotImplementedError
+    end
+
+    # Maximum constexpr recursion depth supported by the compiler.
+    def constexpr_depth
+      raise NotImplementedError
+    end
+
+    # Show the name and the version of the compiler.
+    def to_s
+      raise NotImplementedError
+    end
+
+    # compile_file: Path -> Hash
+    #
+    # Compile the given file and return compilation statistics.
+    #
+    # Additional compiler-specific arguments may be specified.
+    #
+    # A `CompilationError` is be raised if the compilation fails for
+    # whatever reason. Either this method or `compile_code` must be
+    # implemented in subclasses.
+    def compile_file(file, *args)
+      raise ArgumentError, "invalid filename #{file}" unless File.file? file
+      code = File.read(file)
+      compile_code(code, *args)
+    end
+
+    # compile_code: String -> Hash
+    #
+    # Compile the given string and return compilation statistics.
+    #
+    # This method has the same behavior as `compile_file`, except the code
+    # is given as-is instead of being in a file. Either this method or
+    # `compile_file` must be implemented in subclasses.
+    def compile_code(code, *args)
+      tmp = Tempfile.new(["", '.cpp'])
+      tmp.write(code)
+      tmp.close
+      compile_file(tmp.path, *args)
+    ensure
+      tmp.unlink
+    end
+  end # class Compiler
+
+  class Clang < Compiler
+    def initialize(binary)
+      @exe = `which #{binary}`.strip
+      raise "#{binary} not found" unless $?.success?
+    end
+
+    def compile_file(file, *args)
+      file = Pathname.new(file).expand_path
+      command = "time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
+      stdout, stderr, status = Open3.capture3(command)
+      raise CompilationError.new(command, file.read, stderr) unless status.success?
+
+      return {
+        peak_memusg: stderr.match(/(\d+)\s+maximum/)[1].to_i,
+        wall_time: stderr.match(/.+Total/).to_s.split[-3].to_f
+      }
+    end
+
+    def template_depth;  256; end
+    def constexpr_depth; 512; end
+  end
+
+  class GCC < Compiler
+    def initialize(binary)
+      @exe = `which #{binary}`.strip
+      raise "#{binary} not found" unless $?.success?
+    end
+
+    def compile_file(file, *args)
+      file = Pathname.new(file).expand_path
+      command = "time -l #{@exe} #{args.join(' ')} -ftime-report #{file}"
+      stdout, stderr, status = Open3.capture3(command)
+      raise CompilationError.new(command, file.read, stderr) unless status.success?
+
+      return {
+        peak_memusg: stderr.match(/(\d+)\s+maximum/)[1].to_i,
+        wall_time: stderr.match(/TOTAL.+/).to_s.split[-3].to_f
+      }
+    end
+
+    def template_depth;  900; end
+    def constexpr_depth; 512; end
+  end
+
+  class Compiler
+    def self.guess_from_binary(binary)
+      stdout, stderr, status = Open3.capture3("#{binary} --version")
+      case stdout
+      when /\(GCC\)/
+        return GCC.new(binary)
+      when /clang/
+        return Clang.new(binary)
+      else
+        raise ArgumentError("unknown compiler #{binary}")
+      end
+    end
+  end
+end

data/lib/benchcc/ext/enumerable.rb

@@ -0,0 +1,7 @@
+module Enumerable
+  def foldr(state, &block)
+    reverse.inject(state) { |state, x| block.call(x, state) }
+  end
+
+  alias_method(:foldl, :inject)
+end

data/lib/benchcc/ext/file.rb

@@ -0,0 +1,16 @@
+require 'pathname'
+
+
+class File
+  def self.remove_ext(path)
+    path.chomp(extname(path))
+  end
+
+  def self.sub_ext(path, ext)
+    Pathname.new(path).sub_ext(ext).to_path
+  end
+
+  def self.basename_we(path)
+    File.basename(File.remove_ext(path))
+  end
+end

data/lib/benchcc/ext/gnuplot.rb

@@ -0,0 +1,8 @@
+require 'gnuplot'
+
+
+class Gnuplot::DataSet
+  def self.from_file(file, using: [1, 2], &block)
+    self.new("\"#{file}\" using #{using.join(':')}", &block)
+  end
+end

data/lib/benchcc/ext/numeric.rb

@@ -0,0 +1,15 @@
+class Numeric
+  # round_up: Numeric -> Numeric
+  #
+  # Round up the integer to a given precision in decimal digits (default 0
+  # digits). This is similar to `round`, except that rounding is always done
+  # upwards.
+  def round_up(ndigits = 0)
+    k = 10 ** ndigits
+    if self % k == 0
+      self
+    else
+      (1 + self/k) * k
+    end
+  end
+end

data/lib/benchcc/ext/string.rb

@@ -0,0 +1,5 @@
+class String
+  def quote(char = '"')
+    char + self + char
+  end
+end

data/lib/benchcc/fusion.rb

@@ -0,0 +1,54 @@
+require_relative 'ext/enumerable'
+require_relative 'ext/numeric'
+require 'delegate'
+
+
+module Benchcc
+  module Fusion
+    class List < SimpleDelegator
+      def includes
+        inc = ["#define FUSION_MAX_LIST_SIZE 50"]
+        inc << headers.map { |hdr| "#include <#{hdr}>" }
+        inc.join("\n")
+      end
+
+      def headers
+        ["boost/fusion/container/list/list.hpp"]
+      end
+
+      def to_s
+        "boost::fusion::list<#{join(', ')}>"
+      end
+    end
+
+    class Cons < SimpleDelegator
+      def includes
+        headers.map { |hdr| "#include <#{hdr}>" }.join("\n")
+      end
+
+      def headers
+        ["boost/fusion/container/list/cons.hpp"]
+      end
+
+      def to_s
+        foldr('boost::fusion::nil_') do |head, tail|
+          "boost::fusion::cons<#{head}, #{tail}>"
+        end
+      end
+    end
+
+    class Vector < SimpleDelegator
+      def includes
+        headers.map { |hdr| "#include <#{hdr}>" }.join("\n")
+      end
+
+      def headers
+        ["boost/fusion/container/vector/vector#{[10, size.round_up(1)].max}.hpp"]
+      end
+
+      def to_s
+        "boost::fusion::vector#{size}<#{join(', ')}>"
+      end
+    end
+  end
+end

data/lib/benchcc/mpl.rb

@@ -0,0 +1,48 @@
+require_relative 'ext/numeric'
+require 'delegate'
+
+
+module Benchcc
+  module MPL
+    class Sequence < SimpleDelegator
+      def includes
+        headers.map { |hdr| "#include <#{hdr}>" }.join("\n")
+      end
+
+      def headers(name)
+        [
+          "boost/mpl/#{name}/#{name}#{[size.round_up(1), 50].min}.hpp",
+          size > 50 ? "boost/mpl/#{name}/aux_/item.hpp" : nil
+        ].compact
+      end
+    end
+
+    class Vector < Sequence
+      def to_s
+        initial, rest = map(&:to_s).take(50), map(&:to_s).drop(50)
+        vectorN = "boost::mpl::vector#{initial.size}<#{initial.join(', ')}>"
+        rest.reduce(vectorN) do |tail, x|
+          "boost::mpl::v_item<#{x}, #{tail}, 0>" # we emulate mpl::push_back
+        end
+      end
+
+      def headers; super 'vector'; end
+    end
+
+    class List < Sequence
+      def to_s
+        tail = map(&:to_s).last(50)
+        init = map(&:to_s).first([size - tail.size, 0].max)
+        listN = "boost::mpl::list#{tail.size}<#{tail.join(', ')}>"
+
+        init.reverse.zip(51..Float::INFINITY).reduce(listN) do |tail, x_size|
+          x, size = x_size
+          # we emulate mpl::push_front
+          "boost::mpl::l_item<boost::mpl::long_<#{size}>, #{x}, #{tail}>"
+        end
+      end
+
+      def headers; super 'list'; end
+    end
+  end
+end

data/lib/benchcc/plot.rb

@@ -0,0 +1,44 @@
+require 'csv'
+require 'gnuplot'
+
+
+module Benchcc
+  def plot_memusg(output, *inputs)
+    Gnuplot.open do |io|
+      Gnuplot::Plot.new(io) do |plot|
+        plot.ylabel   'Memory usage'
+        plot.decimal  "locale 'en_US.UTF-8'"
+        plot.format   'y "%.2e kb"'
+        plot.term     'png'
+        plot.output   output
+        plot.data = inputs.map { |file|
+          csv = CSV.table(file)
+          Gnuplot::DataSet.new([csv[:x], csv[:peak_memusg]]) { |ds|
+            ds.title = file
+            ds.with = 'lines'
+          }
+        }
+      end
+    end
+  end
+  module_function :plot_memusg
+
+  def plot_time(output, *inputs)
+    Gnuplot.open do |io|
+      Gnuplot::Plot.new(io) do |plot|
+        plot.ylabel   'Compilation time'
+        plot.format   'y "%.2g s"'
+        plot.term     'png'
+        plot.output   output
+        plot.data = inputs.map { |file|
+          csv = CSV.table(file)
+          Gnuplot::DataSet.new([csv[:x], csv[:wall_time]]) { |ds|
+            ds.title = file
+            ds.with = 'lines'
+          }
+        }
+      end
+    end
+  end
+  module_function :plot_time
+end

data/lib/benchcc/version.rb

@@ -0,0 +1,3 @@
+module Benchcc
+  VERSION = "0.0.1"
+end

data/lib/benchcc.rb

@@ -0,0 +1,6 @@
+require 'benchcc/benchmark'
+require 'benchcc/compiler'
+require 'benchcc/fusion'
+require 'benchcc/mpl'
+require 'benchcc/plot'
+require 'benchcc/version'

data/spec/benchmark_spec.rb

@@ -0,0 +1,23 @@
+require 'benchcc/benchmark'
+
+require 'rspec'
+require 'tempfile'
+require 'timeout'
+
+
+describe Benchcc.method(:benchmark) do
+  it('time outs are ignored') {
+    timeout = 0.3
+    envs = [{time: timeout + 0.1}, {time: timeout - 0.1},
+            {time: timeout + 0.2}, {time: timeout - 0.2}]
+    Tempfile.create('') do |file|
+      file.write('<%= time %>')
+      file.flush
+      data = Benchcc.benchmark(file.path, envs, timeout: timeout) do |code|
+        sleep(code.to_f)
+        {time: code.to_f}
+      end
+      expect(data).to eq(envs.select { |env| env[:time] < timeout })
+    end
+  }
+end

data/spec/compiler_spec.rb

@@ -0,0 +1,69 @@
+require 'benchcc/compiler'
+
+require 'pathname'
+require 'rspec'
+require 'tempfile'
+
+
+describe Benchcc::Compiler do
+  describe :compile_file do
+    it {
+      class Mock < Benchcc::Compiler
+        def compile_code(code); code; end
+      end
+
+      file = Tempfile.new('')
+      file.write("whatever")
+      expect(Mock.new.compile_file(file.path)).to eq(file.read)
+    }
+  end
+
+  describe :compile_code do
+    it {
+      class Mock < Benchcc::Compiler
+        def compile_file(file); File.read(file); end
+      end
+
+      expect(Mock.new.compile_code("whatever")).to eq("whatever")
+    }
+  end
+
+  describe :guess_from_binary do
+    it('can guess clang') {
+      expect(Benchcc::Compiler.guess_from_binary(`which clang`.strip)).to be_instance_of(Benchcc::Clang)
+    }
+
+    # GCC is aliased to clang on OSX, so this test fails.
+    # it('can guess gcc') {
+    #   expect(Benchcc::Compiler.guess_from_binary(`which gcc`.strip)).to be_instance_of(Benchcc::GCC)
+    # }
+
+    it('fails otherwise') {
+      expect { Benchcc::Compiler.guess_from_binary('not_a_compiler') }.to raise_error
+    }
+  end
+end
+
+[[Benchcc::Clang, 'clang++'], [Benchcc::GCC, 'g++']].each do |compiler, which|
+  describe compiler do
+    before(:each) {
+      @cc = compiler.new(which)
+      @invalid = Pathname.new('src/invalid.cpp').expand_path(File.dirname(__FILE__))
+      @valid = Pathname.new('src/valid.cpp').expand_path(File.dirname(__FILE__))
+    }
+
+    describe :compile_file do
+      it('fails cleanly on invalid input') {
+        expect {
+          @cc.compile_file(@invalid)
+        }.to raise_error(Benchcc::CompilationError)
+      }
+
+      it('returns statistics on valid input') {
+        result = @cc.compile_file(@valid, '-o /dev/null')
+        expect(result).to have_key(:peak_memusg)
+        expect(result).to have_key(:wall_time)
+      }
+    end
+  end
+end

data/spec/numeric_spec.rb

@@ -0,0 +1,23 @@
+require "benchcc/ext/numeric"
+
+require "rspec"
+
+
+describe Numeric do
+  describe :round_up do
+    it {
+      -150.upto(150) do |n|
+        expect(n.round_up).to eq(n)
+      end
+    }
+
+    it {
+      (10..150).step(10) do |tenths|
+        tenths.downto(tenths-9) do |n|
+          expect(n.round_up(1)).to eq(tenths)
+        end
+        expect((tenths-10).round_up(1)).to eq(tenths-10)
+      end
+    }
+  end
+end

data/spec/plot_spec.rb

@@ -0,0 +1,29 @@
+require 'benchcc/plot'
+
+require 'fileutils'
+require 'rspec'
+
+
+describe 'plots' do
+  before :each do
+    @tmpdir = Dir.mktmpdir
+    @out = File.join(@tmpdir, 'plot.png')
+    @csv = Pathname.new('src/plot.csv').expand_path(File.dirname(__FILE__))
+  end
+
+  after :each do
+    FileUtils.remove_entry(@tmpdir)
+  end
+
+  describe Benchcc.method(:plot_memusg) do
+    it {
+      expect { Benchcc.plot_memusg(@out, @csv) }.not_to raise_error
+    }
+  end
+
+  describe Benchcc.method(:plot_time) do
+    it {
+      expect { Benchcc.plot_time(@out, @csv) }.not_to raise_error
+    }
+  end
+end

data/spec/src/invalid.cpp

@@ -0,0 +1 @@
+int main() {

data/spec/src/plot.csv

@@ -0,0 +1,3 @@
+x, peak_memusg, wall_time
+0, 1,         , 2
+1, 2,         , 3

data/spec/src/valid.cpp

@@ -0,0 +1 @@
+int main() {}