benchmark_driver 0.8.6 → 0.9.0

data/lib/benchmark_driver/output/simple.rb

@@ -0,0 +1,97 @@
+class BenchmarkDriver::Output::Simple
+  NAME_LENGTH = 8
+
+  # @param [Array<BenchmarkDriver::*::Job>] jobs
+  # @param [Array<BenchmarkDriver::Config::Executable>] executables
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  def initialize(jobs:, executables:, metrics_type:)
+    @jobs = jobs
+    @executables = executables
+    @metrics_type = metrics_type
+    @name_length = jobs.map { |j| j.name.size }.max
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.print 'warming up'
+      block.call
+    end
+  ensure
+    $stdout.puts
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_benchmark(&block)
+    @with_benchmark = true
+    without_stdout_buffering do
+      # Show header
+      $stdout.puts "benchmark results (#{@metrics_type.unit}):"
+
+      # Show executable names
+      if @executables.size > 1
+        $stdout.print("#{' ' * @name_length}  ")
+        @executables.each do |executable|
+          $stdout.print("%#{NAME_LENGTH}s  " % executable.name)
+        end
+        $stdout.puts
+      end
+
+      block.call
+    end
+  rescue
+    @with_benchmark = false
+  end
+
+  # @param [BenchmarkDriver::*::Job] job
+  def with_job(job, &block)
+    if @with_benchmark
+      $stdout.print("%-#{@name_length}s  " % job.name)
+    end
+    block.call
+  ensure
+    if @with_benchmark
+      $stdout.puts
+    end
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def report(metrics)
+    if @with_benchmark
+      $stdout.print("%#{NAME_LENGTH}s  " % humanize(metrics.value))
+    else
+      $stdout.print '.'
+    end
+  end
+
+  private
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value)
+    if value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    prefix = "%6.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        scale = 0
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+end

data/lib/benchmark_driver/rbenv.rb

@@ -0,0 +1,11 @@
+module BenchmarkDriver
+  module Rbenv
+    def self.ruby_path(version)
+      path = `RBENV_VERSION='#{version}' rbenv which ruby`.rstrip
+      unless $?.success?
+        abort "Failed to execute 'rbenv which ruby'"
+      end
+      path
+    end
+  end
+end

data/lib/benchmark_driver/ruby_interface.rb

@@ -0,0 +1,51 @@
+module BenchmarkDriver
+  class RubyInterface
+    def self.run(**args, &block)
+      new(**args).tap { |x| block.call(x) }.run
+    end
+
+    # Build jobs and run. This is NOT interface for users.
+    def run
+      jobs = @jobs.flat_map do |job|
+        BenchmarkDriver::JobParser.parse({
+          type: @config.runner_type,
+          prelude: @prelude,
+        }.merge!(job))
+      end
+      BenchmarkDriver::Runner.run(jobs, config: @config)
+    end
+
+    #
+    # Config APIs from here
+    #
+
+    # @param [String,NilClass] output
+    # @param [String,NilClass] runner
+    def initialize(output: nil, runner: nil)
+      @prelude = ''
+      @jobs = []
+      @config = BenchmarkDriver::Config.new
+      @config.output_type = output.to_s if output
+      @config.runner_type = runner.to_s if runner
+    end
+
+    # @param [String] script
+    def prelude(script)
+      @prelude << "#{script}\n"
+    end
+
+    # @param [String] name - Name shown on result output.
+    # @param [String,nil] script - Benchmarked script in String. If nil, name is considered as script too.
+    def report(name, script = nil)
+      if script.nil?
+        script = name
+      end
+      @jobs << { benchmark: [{ name: name, script: script }] }
+    end
+
+    # Backward compatibility. This is actually default now.
+    def compare!
+      @config.output_type = 'compare'
+    end
+  end
+end

data/lib/benchmark_driver/runner.rb

@@ -0,0 +1,42 @@
+module BenchmarkDriver
+  module Runner
+    require 'benchmark_driver/runner/ips'
+    require 'benchmark_driver/runner/memory'
+    require 'benchmark_driver/runner/time'
+  end
+
+  class << Runner
+    # Main function which is used by both CLI and `Benchmark.driver`.
+    # @param [Array<BenchmarkDriver::*::Job>] jobs
+    # @param [BenchmarkDriver::Config] config
+    def run(jobs, config:)
+      runner_config = Config::RunnerConfig.new(
+        executables: config.executables,
+        repeat_count: config.repeat_count,
+        run_duration: config.run_duration,
+      )
+
+      jobs.group_by(&:class).each do |klass, jobs_group|
+        runner = runner_for(klass)
+        output = Output.find(config.output_type).new(
+          jobs: jobs,
+          executables: config.executables,
+          metrics_type: runner::MetricsType,
+        )
+        runner.new(config: runner_config, output: output).run(jobs)
+      end
+    end
+
+    private
+
+    # Dynamically find class (BenchmarkDriver::*::JobRunner) for plugin support
+    # @param [Class] klass - BenchmarkDriver::*::Job
+    # @return [Class]
+    def runner_for(klass)
+      unless match = klass.name.match(/\ABenchmarkDriver::Runner::(?<namespace>[^:]+)::Job\z/)
+        raise "Unexpected job class: #{klass}"
+      end
+      BenchmarkDriver.const_get("Runner::#{match[:namespace]}", false)
+    end
+  end
+end

data/lib/benchmark_driver/runner/ips.rb

@@ -0,0 +1,239 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metrics'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+require 'shellwords'
+
+class BenchmarkDriver::Runner::Ips
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
+  # Passed to `output` by `BenchmarkDriver::Runner.run`
+  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 'i/s')
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output::*] output
+  def initialize(config:, output:)
+    @config = config
+    @output = output
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    if jobs.any? { |job| job.loop_count.nil? }
+      @output.with_warmup do
+        jobs = jobs.map do |job|
+          next job if job.loop_count # skip warmup if loop_count is set
+
+          @output.with_job(job) do
+            result = run_warmup(job, exec: @config.executables.first)
+            metrics = build_metrics(result)
+            @output.report(metrics)
+
+            loop_count = (result.fetch(:loop_count).to_f * @config.run_duration / result.fetch(:duration)).floor
+            Job.new(job.to_h.merge(loop_count: loop_count))
+          end
+        end
+      end
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(job) do
+          @config.executables.each do |exec|
+            best_metrics = with_repeat(@config.repeat_count) do
+              run_benchmark(job, exec: exec)
+            end
+            @output.report(best_metrics)
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is nil
+  # @param [BenchmarkDriver::Config::Executable] exec
+  def run_warmup(job, exec:)
+    warmup = WarmupScript.new(
+      prelude:    job.prelude,
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+      first_warmup_duration: @config.run_duration / 6.0,  # default: 0.5
+      second_warmup_duration: @config.run_duration / 3.0, # default: 1.0
+    )
+
+    hash = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(warmup.render(result: f.path)) do |path|
+        execute(*exec.command, path)
+      end
+      eval(f.read)
+    end
+
+    hash.merge(executable: exec)
+  end
+
+  # Return multiple times and return the best metrics
+  def with_repeat(repeat_times, &block)
+    all_metrics = repeat_times.times.map do
+      block.call
+    end
+    all_metrics.sort_by do |metrics|
+      metrics.value
+    end.last
+  end
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Config::Executable] exec
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, exec:)
+    benchmark = BenchmarkScript.new(
+      prelude:    job.prelude,
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render(result: f.path)) do |path|
+        execute(*exec.command, path)
+      end
+      Float(f.read)
+    end
+
+    build_metrics(
+      loop_count: job.loop_count,
+      duration: duration,
+      executable: exec,
+    )
+  end
+
+  # This method is overridden by BenchmarkDriver::Runner::Time
+  def build_metrics(duration:, executable:, loop_count:)
+    BenchmarkDriver::Metrics.new(
+      value: loop_count.to_f / duration,
+      duration: duration,
+      executable: executable,
+    )
+  end
+
+  def with_script(script)
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  def execute(*args)
+    IO.popen(args, &:read) # handle stdout?
+    unless $?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+    end
+  end
+
+  WarmupScript = ::BenchmarkDriver::Struct.new(:prelude, :script, :teardown, :loop_count, :first_warmup_duration, :second_warmup_duration) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      <<-RUBY
+#{prelude}
+
+# first warmup
+__bmdv_i = 0
+__bmdv_before = Time.now
+__bmdv_target = __bmdv_before + #{first_warmup_duration}
+while Time.now < __bmdv_target
+  #{script}
+  __bmdv_i += 1
+end
+__bmdv_after = Time.now
+
+# second warmup
+__bmdv_ip100ms = (__bmdv_i.to_f / (__bmdv_after - __bmdv_before) / 10.0).floor
+__bmdv_loops = 0
+__bmdv_duration = 0.0
+__bmdv_target = Time.now + #{second_warmup_duration}
+while Time.now < __bmdv_target
+  __bmdv_i = 0
+  __bmdv_before = Time.now
+  while __bmdv_i < __bmdv_ip100ms
+    #{script}
+    __bmdv_i += 1
+  end
+  __bmdv_after = Time.now
+
+  __bmdv_loops += __bmdv_i
+  __bmdv_duration += (__bmdv_after - __bmdv_before)
+end
+
+#{teardown}
+
+File.write(#{result.dump}, { duration: __bmdv_duration, loop_count: __bmdv_loops }.inspect)
+      RUBY
+    end
+  end
+  private_constant :WarmupScript
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:prelude, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      <<-RUBY
+#{prelude}
+
+if Process.respond_to?(:clock_gettime) # Ruby 2.1+
+  __bmdv_empty_before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  #{while_loop('', loop_count)}
+  __bmdv_empty_after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+else
+  __bmdv_empty_before = Time.now
+  #{while_loop('', loop_count)}
+  __bmdv_empty_after = Time.now
+end
+
+if Process.respond_to?(:clock_gettime) # Ruby 2.1+
+  __bmdv_script_before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  #{while_loop(script, loop_count)}
+  __bmdv_script_after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+else
+  __bmdv_script_before = Time.now
+  #{while_loop(script, loop_count)}
+  __bmdv_script_after = Time.now
+end
+
+#{teardown}
+
+File.write(
+  #{result.dump},
+  ((__bmdv_script_after - __bmdv_script_before) - (__bmdv_empty_after - __bmdv_empty_before)).inspect,
+)
+      RUBY
+    end
+
+    private
+
+    def while_loop(content, times)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      end
+
+      # TODO: execute in batch
+      <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+  #{content}
+  __bmdv_i += 1
+end
+      RUBY
+    end
+  end
+  private_constant :BenchmarkScript
+end

data/lib/benchmark_driver/runner/memory.rb

@@ -0,0 +1,142 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metrics'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+require 'shellwords'
+
+# Max resident set size
+class BenchmarkDriver::Runner::Memory
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(Job)
+  # Passed to `output` by `BenchmarkDriver::Runner.run`
+  MetricsType = BenchmarkDriver::Metrics::Type.new(unit: 'bytes', larger_better: false, worse_word: 'larger')
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output::*] output
+  def initialize(config:, output:)
+    @config = config
+    @output = output
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Default::Job>] jobs
+  def run(jobs)
+    # Currently Linux's time(1) support only...
+    if Etc.uname.fetch(:sysname) != 'Linux'
+      raise "memory output is not supported for '#{Etc.uname[:sysname]}' for now"
+    end
+
+    if jobs.any? { |job| job.loop_count.nil? }
+      jobs = jobs.map do |job|
+        job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))
+      end
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(job) do
+          @config.executables.each do |exec|
+            best_metrics = with_repeat(@config.repeat_count) do
+              run_benchmark(job, exec: exec)
+            end
+            @output.report(best_metrics)
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # Return multiple times and return the best metrics
+  def with_repeat(repeat_times, &block)
+    all_metrics = repeat_times.times.map do
+      block.call
+    end
+    all_metrics.sort_by do |metrics|
+      metrics.value
+    end.first
+  end
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Config::Executable] exec
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, exec:)
+    benchmark = BenchmarkScript.new(
+      prelude:    job.prelude,
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    output = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render(result: f.path)) do |path|
+        execute('/usr/bin/time', *exec.command, path)
+      end
+    end
+
+    match_data = /^(?<user>\d+.\d+)user\s+(?<system>\d+.\d+)system\s+(?<elapsed1>\d+):(?<elapsed2>\d+.\d+)elapsed.+\([^\s]+\s+(?<maxresident>\d+)maxresident\)k$/.match(output)
+    raise "Unexpected format given from /usr/bin/time:\n#{out}" unless match_data[:maxresident]
+
+    BenchmarkDriver::Metrics.new(
+      value: Integer(match_data[:maxresident]) * 1000.0, # kilobytes -> bytes
+      executable: exec,
+    )
+  end
+
+  def with_script(script)
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  def execute(*args)
+    output = IO.popen(args, err: [:child, :out], &:read) # handle stdout?
+    unless $?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+    end
+    output
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:prelude, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      <<-RUBY
+#{prelude}
+#{while_loop('', loop_count)}
+#{teardown}
+      RUBY
+    end
+
+    private
+
+    def while_loop(content, times)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      end
+
+      # TODO: execute in batch
+      if times > 1
+        <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+  #{content}
+  __bmdv_i += 1
+end
+        RUBY
+      else
+        content
+      end
+    end
+  end
+  private_constant :BenchmarkScript
+end