benchmark_driver 0.8.6 → 0.9.0

data/lib/benchmark_driver.rb

@@ -1 +1,12 @@
-require 'benchmark/driver'
+require 'benchmark_driver/config'
+require 'benchmark_driver/job_parser'
+require 'benchmark_driver/output'
+require 'benchmark_driver/rbenv'
+require 'benchmark_driver/ruby_interface'
+require 'benchmark_driver/runner'
+require 'benchmark_driver/version'
+
+require 'benchmark'
+def Benchmark.driver(**args, &block)
+  BenchmarkDriver::RubyInterface.run(**args, &block)
+end

data/lib/benchmark_driver/config.rb

@@ -0,0 +1,36 @@
+require 'benchmark_driver/struct'
+
+module BenchmarkDriver
+  # All CLI options
+  Config = ::BenchmarkDriver::Struct.new(
+    :runner_type,  # @param [String]
+    :output_type,  # @param [String]
+    :paths,        # @param [Array<String>]
+    :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
+    :filters,      # @param [Array<Regexp>]
+    :repeat_count, # @param [Integer]
+    :run_duration, # @param [Integer]
+    defaults: {
+      runner_type: 'ips',
+      output_type: 'compare',
+      filters: [],
+      repeat_count: 1,
+      run_duration: 3,
+    },
+  )
+
+  # Subset of FullConfig passed to JobRunner
+  Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
+    :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
+    :repeat_count, # @param [Integer]
+    :run_duration, # @param [Integer]
+  )
+
+  Config::Executable = ::BenchmarkDriver::Struct.new(
+    :name,    # @param [String]
+    :command, # @param [Array<String>]
+  )
+  Config.defaults[:executables] = [
+    BenchmarkDriver::Config::Executable.new(name: RUBY_VERSION, command: [RbConfig.ruby]),
+  ]
+end

data/lib/benchmark_driver/default_job.rb

@@ -0,0 +1,12 @@
+require 'benchmark_driver/struct'
+
+module BenchmarkDriver
+  DefaultJob = ::BenchmarkDriver::Struct.new(
+    :name,       # @param [String] name
+    :script,     # @param [String] benchmark
+    :prelude,    # @param [String,nil] prelude (optional)
+    :teardown,   # @param [String,nil] after (optional)
+    :loop_count, # @param [Integer,nil] loop_count (optional)
+    defaults: { prelude: '', teardown: '' },
+  )
+end

data/lib/benchmark_driver/default_job_parser.rb

@@ -0,0 +1,68 @@
+module BenchmarkDriver
+  module DefaultJobParser
+    # Build default JobParser for given job klass
+    def self.for(klass)
+      Module.new.tap do |parser|
+        class << parser
+          include DefaultJobParser
+        end
+        parser.define_singleton_method(:job_class) do
+          klass
+        end
+      end
+    end
+
+    # This method is dynamically called by `BenchmarkDriver::JobParser.parse`
+    # @param [String] prelude
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    # @param [String] teardown
+    # @param [Integer] loop_count
+    # @return [Array<BenchmarkDriver::Default::Job>]
+    def parse(prelude: nil, benchmark:, teardown: nil, loop_count: nil)
+      parse_benchmark(benchmark).each do |job|
+        job.prelude.prepend("#{prelude}\n") if prelude
+        job.teardown.prepend("#{teardown}\n") if teardown
+        job.loop_count ||= loop_count
+      end.each(&:freeze)
+    end
+
+    private
+
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    def parse_benchmark(benchmark)
+      case benchmark
+      when String
+        [parse_job(benchmark)]
+      when Array
+        benchmark.map { |b| parse_job(b) }
+      when Hash
+        benchmark.map do |key, value|
+          job_class.new(name: key.to_s, script: value)
+        end
+      else
+        raise ArgumentError.new("benchmark must be String, Array or Hash, but got: #{benchmark.inspect}")
+      end
+    end
+
+    # @param [String,Hash{ Symbol => String }>] bench
+    def parse_job(benchmark)
+      case benchmark
+      when String
+        job_class.new(name: benchmark, script: benchmark)
+      when Hash
+        parse_job_hash(benchmark)
+      else
+        raise ArgumentError.new("Expected String or Hash in element of benchmark, but got: #{benchmark.inspect}")
+      end
+    end
+
+    def parse_job_hash(name: nil, prelude: '', script:, teardown: '', loop_count: nil)
+      name ||= script
+      job_class.new(name: name, prelude: prelude, script: script, teardown: teardown, loop_count: loop_count)
+    end
+
+    def job_class
+      raise NotImplementedError # override this
+    end
+  end
+end

data/lib/benchmark_driver/job_parser.rb

@@ -0,0 +1,42 @@
+require 'benchmark_driver/runner'
+
+module BenchmarkDriver
+  class << JobParser = Module.new
+    # @param [Hash] config
+    def parse(config)
+      config = symbolize_keys(config)
+      type = config.fetch(:type)
+      if !type.is_a?(String)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
+      elsif !type.match(/\A[A-Za-z0-9_]+\z/)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
+      end
+      config.delete(:type)
+
+      # Dynamic dispatch for plugin support
+      ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(config)
+    end
+
+    private
+
+    def camelize(str)
+      str.split('_').map(&:capitalize).join
+    end
+
+    # @param [Object] config
+    def symbolize_keys(config)
+      case config
+      when Hash
+        config.dup.tap do |hash|
+          hash.keys.each do |key|
+            hash[key.to_sym] = symbolize_keys(hash.delete(key))
+          end
+        end
+      when Array
+        config.map { |c| symbolize_keys(c) }
+      else
+        config
+      end
+    end
+  end
+end

data/lib/benchmark_driver/metrics.rb

@@ -0,0 +1,17 @@
+require 'benchmark_driver/struct'
+
+# All benchmark results should be expressed by this model.
+module BenchmarkDriver
+  Metrics = ::BenchmarkDriver::Struct.new(
+    :value,      # @param [Float] - The main field of benchmark result
+    :executable, # @param [BenchmarkDriver::Config::Executable] - Measured Ruby executable
+    :duration,   # @param [Float,nil] - Time taken to run the script (optional)
+  )
+
+  Metrics::Type = ::BenchmarkDriver::Struct.new(
+    :unit,          # @param [String] - A label of unit for the value.
+    :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+    :worse_word,    # @param [String] - A label shown when the value is worse.
+    defaults: { larger_better: true, worse_word: 'slower' },
+  )
+end

data/lib/benchmark_driver/output.rb

@@ -0,0 +1,27 @@
+module BenchmarkDriver
+  module Output
+    require 'benchmark_driver/output/compare'
+    require 'benchmark_driver/output/markdown'
+    require 'benchmark_driver/output/simple'
+  end
+
+  class << Output
+    # BenchmarkDriver::Output is pluggable.
+    # Create `BenchmarkDriver::Output::Foo` as benchmark_dirver-output-foo.gem and specify `-o foo`.
+    #
+    # @param [String] type
+    def find(type)
+      if type.include?(':')
+        raise ArgumentError.new("Output type '#{type}' cannot contain ':'")
+      end
+
+      ::BenchmarkDriver::Output.const_get(camelize(type), false)
+    end
+
+    private
+
+    def camelize(str)
+      str.split('_').map(&:capitalize).join
+    end
+  end
+end

data/lib/benchmark_driver/output/compare.rb

@@ -0,0 +1,196 @@
+# Compare output like benchmark-ips
+class BenchmarkDriver::Output::Compare
+  NAME_LENGTH = 20
+
+  # @param [Array<BenchmarkDriver::*::Job>] jobs
+  # @param [Array<BenchmarkDriver::Config::Executable>] executables
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  def initialize(jobs:, executables:, metrics_type:)
+    @jobs = jobs
+    @executables = executables
+    @metrics_type = metrics_type
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.puts 'Warming up --------------------------------------'
+      # TODO: show exec name if it has multiple ones
+      block.call
+    end
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_benchmark(&block)
+    @metrics_by_job = Hash.new { |h, k| h[k] = [] }
+
+    without_stdout_buffering do
+      $stdout.puts 'Calculating -------------------------------------'
+      if @executables.size > 1
+        $stdout.print(' ' * NAME_LENGTH)
+        @executables.each do |executable|
+          $stdout.print(' %10s ' % executable.name)
+        end
+        $stdout.puts
+      end
+
+      block.call
+    end
+  ensure
+    if @executables.size > 1
+      compare_executables
+    elsif @jobs.size > 1
+      compare_jobs
+    end
+  end
+
+  # @param [BenchmarkDriver::*::Job] job
+  def with_job(job, &block)
+    if job.name.length > NAME_LENGTH
+      $stdout.puts(job.name)
+    else
+      $stdout.print("%#{NAME_LENGTH}s" % job.name)
+    end
+    @current_job = job
+    @job_metrics = []
+    block.call
+  ensure
+    $stdout.print(@metrics_type.unit)
+    if job.loop_count
+      $stdout.print(" - #{humanize(job.loop_count)} times")
+      if @job_metrics.all? { |metrics| metrics.duration }
+        $stdout.print(" in")
+        show_durations
+      end
+    end
+    $stdout.puts
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def report(metrics)
+    if defined?(@metrics_by_job)
+      @metrics_by_job[@current_job] << metrics
+    end
+
+    @job_metrics << metrics
+    $stdout.print("#{humanize(metrics.value, [10, metrics.executable.name.length].max)} ")
+  end
+
+  private
+
+  def show_durations
+    @job_metrics.each do |metrics|
+      $stdout.print(' %3.6fs' % metrics.duration)
+    end
+
+    # Show pretty seconds / clocks too. As it takes long width, it's shown only with a single executable.
+    if @job_metrics.size == 1
+      metrics = @job_metrics.first
+      sec = metrics.duration
+      iter = @current_job.loop_count
+      if File.exist?('/proc/cpuinfo') && (clks = estimate_clock(sec, iter)) < 1_000
+        $stdout.print(" (#{pretty_sec(sec, iter)}/i, #{clks}clocks/i)")
+      else
+        $stdout.print(" (#{pretty_sec(sec, iter)}/i)")
+      end
+    end
+  end
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value, width = 10)
+    if value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    prefix = "%#{width}.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        scale = 0
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+
+  def pretty_sec(sec, iter)
+    r = Rational(sec, iter)
+    case
+    when r >= 1
+      "#{'%3.2f' % r.to_f}s"
+    when r >= 1/1000r
+      "#{'%3.2f' % (r * 1_000).to_f}ms"
+    when r >= 1/1000_000r
+      "#{'%3.2f' % (r * 1_000_000).to_f}μs"
+    else
+      "#{'%3.2f' % (r * 1_000_000_000).to_f}ns"
+    end
+  end
+
+  def estimate_clock sec, iter
+    hz = File.read('/proc/cpuinfo').scan(/cpu MHz\s+:\s+([\d\.]+)/){|(f)| break hz = Rational(f.to_f) * 1_000_000}
+    r = Rational(sec, iter)
+    Integer(r/(1/hz))
+  end
+
+  def compare_jobs
+    $stdout.puts "\nComparison:"
+    results = @metrics_by_job.map { |job, metrics| Result.new(job: job, metrics: metrics.first) }
+    show_results(results, show_executable: false)
+  end
+
+  def compare_executables
+    $stdout.puts "\nComparison:"
+
+    @metrics_by_job.each do |job, metrics|
+      $stdout.puts("%#{NAME_LENGTH + 2 + 11}s" % job.name)
+      results = metrics.map { |metrics| Result.new(job: job, metrics: metrics) }
+      show_results(results, show_executable: true)
+    end
+  end
+
+  # @param [Array<BenchmarkDriver::Output::Compare::Result>] results
+  # @param [TrueClass,FalseClass] show_executable
+  def show_results(results, show_executable:)
+    results = results.sort_by do |result|
+      if @metrics_type.larger_better
+        -result.metrics.value
+      else
+        result.metrics.value
+      end
+    end
+
+    first = results.first
+    results.each do |result|
+      if result != first
+        if @metrics_type.larger_better
+          ratio = (first.metrics.value / result.metrics.value)
+        else
+          ratio = (result.metrics.value / first.metrics.value)
+        end
+        slower = "- %.2fx  #{@metrics_type.worse_word}" % ratio
+      end
+      if show_executable
+        name = result.metrics.executable.name
+      else
+        name = result.job.name
+      end
+      $stdout.puts("%#{NAME_LENGTH}s: %11.1f %s #{slower}" % [name, result.metrics.value, @metrics_type.unit])
+    end
+    $stdout.puts
+  end
+
+  Result = ::BenchmarkDriver::Struct.new(:job, :metrics)
+end

data/lib/benchmark_driver/output/markdown.rb

@@ -0,0 +1,102 @@
+class BenchmarkDriver::Output::Markdown
+  NAME_LENGTH = 8
+
+  # @param [Array<BenchmarkDriver::*::Job>] jobs
+  # @param [Array<BenchmarkDriver::Config::Executable>] executables
+  # @param [BenchmarkDriver::Metrics::Type] metrics_type
+  def initialize(jobs:, executables:, metrics_type:)
+    @jobs = jobs
+    @executables = executables
+    @metrics_type = metrics_type
+    @name_length = jobs.map { |j| j.name.size }.max
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.print 'warming up'
+      block.call
+    end
+  ensure
+    $stdout.puts
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def with_benchmark(&block)
+    @with_benchmark = true
+    without_stdout_buffering do
+      # Show header
+      $stdout.puts "# benchmark results (#{@metrics_type.unit})\n\n"
+
+      # Show executable names
+      $stdout.print("|#{' ' * @name_length}  ")
+      @executables.each do |executable|
+        $stdout.print("|%#{NAME_LENGTH}s" % executable.name) # same size as humanize
+      end
+      $stdout.puts('|')
+
+      # Show header separator
+      $stdout.print("|:#{'-' * (@name_length - 1)}--")
+      @executables.each do |executable|
+        $stdout.print("|:#{'-' * (NAME_LENGTH - 1)}") # same size as humanize
+      end
+      $stdout.puts('|')
+
+      block.call
+    end
+  rescue
+    @with_benchmark = false
+  end
+
+  # @param [BenchmarkDriver::*::Job] job
+  def with_job(job, &block)
+    if @with_benchmark
+      $stdout.print("|%-#{@name_length}s  " % job.name)
+    end
+    block.call
+  ensure
+    if @with_benchmark
+      $stdout.puts('|')
+    end
+  end
+
+  # @param [BenchmarkDriver::Metrics] metrics
+  def report(metrics)
+    if @with_benchmark
+      $stdout.print("|%#{NAME_LENGTH}s" % humanize(metrics.value))
+    else
+      $stdout.print '.'
+    end
+  end
+
+  private
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value)
+    if value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    prefix = "%6.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        scale = 0
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+end