benchmark_driver_monotonic_raw 0.14.13

data/lib/benchmark_driver/output/markdown.rb

@@ -0,0 +1,107 @@
+class BenchmarkDriver::Output::Markdown
+  NAME_LENGTH = 8
+
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @context_names = contexts.map(&:name)
+    @name_length = jobs.map(&:name).map(&:size).max
+  end
+
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.print 'warming up'
+      block.call
+    end
+  ensure
+    $stdout.puts
+  end
+
+  def with_benchmark(&block)
+    @with_benchmark = true
+    without_stdout_buffering do
+      # Show header
+      $stdout.puts "# #{@metrics.first.name} (#{@metrics.first.unit})\n\n"
+
+      # Show executable names
+      $stdout.print("|#{' ' * @name_length}  ")
+      @context_names.each do |context_name|
+        $stdout.print("|%#{NAME_LENGTH}s" % context_name) # same size as humanize
+      end
+      $stdout.puts('|')
+
+      # Show header separator
+      $stdout.print("|:#{'-' * (@name_length - 1)}--")
+      @context_names.each do |context_name|
+        $stdout.print("|:#{'-' * (NAME_LENGTH - 1)}") # same size as humanize
+      end
+      $stdout.puts('|')
+
+      block.call
+    end
+  rescue
+    @with_benchmark = false
+  end
+
+  # @param [BenchmarkDriver::Job] job
+  def with_job(job, &block)
+    if @with_benchmark
+      $stdout.print("|%-#{@name_length}s  " % job.name)
+    end
+    block.call
+  ensure
+    if @with_benchmark
+      $stdout.puts('|')
+    end
+  end
+
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    if @with_benchmark
+      $stdout.print("|%#{NAME_LENGTH}s" % humanize(result.values.fetch(@metrics.first)))
+    else
+      $stdout.print '.'
+    end
+  end
+
+  private
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value)
+    if BenchmarkDriver::Result::ERROR.equal?(value)
+      return " %#{NAME_LENGTH}s" % 'ERROR'
+    elsif value == 0.0
+      return " %#{NAME_LENGTH}.3f" % 0.0
+    elsif value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    prefix = "%#{NAME_LENGTH}.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+end

data/lib/benchmark_driver/output/record.rb

@@ -0,0 +1,61 @@
+class BenchmarkDriver::Output::Record
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @job_warmup_context_result = Hash.new do |h1, job|
+      h1[job] = Hash.new do |h2, warmup|
+        h2[warmup] = Hash.new do |h3, context|
+          h3[context] = {}
+        end
+      end
+    end
+  end
+
+  def with_warmup(&block)
+    $stdout.print 'warming up'
+    block.call
+  ensure
+    $stdout.puts
+  end
+
+  def with_benchmark(&block)
+    @with_benchmark = true
+    $stdout.print 'benchmarking'
+    block.call
+  ensure
+    $stdout.puts
+    @with_benchmark = false
+    save_record
+  end
+
+  # @param [BenchmarkDriver::Job] job
+  def with_job(job, &block)
+    @job = job
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    @context = context
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    $stdout.print '.'
+    @job_warmup_context_result[@job][!@with_benchmark][@context] = result
+  end
+
+  private
+
+  def save_record
+    yaml = {
+      'type' => 'recorded',
+      'job_warmup_context_result' => @job_warmup_context_result,
+      'metrics' => @metrics,
+    }.to_yaml
+    File.write('benchmark_driver.record.yml', yaml)
+  end
+end

data/lib/benchmark_driver/output/simple.rb

@@ -0,0 +1,103 @@
+class BenchmarkDriver::Output::Simple
+  NAME_LENGTH = 10
+
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @context_names = contexts.map(&:name)
+    @name_length = jobs.map(&:name).map(&:size).max
+  end
+
+  def with_warmup(&block)
+    @with_benchmark = false
+    without_stdout_buffering do
+      $stdout.print 'warming up'
+      block.call
+    end
+  ensure
+    $stdout.puts
+  end
+
+  def with_benchmark(&block)
+    @with_benchmark = true
+    without_stdout_buffering do
+      # Show header
+      $stdout.puts "#{@metrics.first.name} (#{@metrics.first.unit}):"
+
+      # Show executable names
+      if @context_names.size > 1
+        $stdout.print("#{' ' * @name_length}  ")
+        @context_names.each do |context_name|
+          $stdout.print("%#{NAME_LENGTH}s  " % context_name)
+        end
+        $stdout.puts
+      end
+
+      block.call
+    end
+  ensure
+    @with_benchmark = false
+  end
+
+  # @param [BenchmarkDriver::Job] job
+  def with_job(job, &block)
+    if @with_benchmark
+      $stdout.print("%-#{@name_length}s  " % job.name)
+    end
+    block.call
+  ensure
+    if @with_benchmark
+      $stdout.puts
+    end
+  end
+
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    if @with_benchmark
+      $stdout.print("%#{NAME_LENGTH}s  " % humanize(result.values.fetch(@metrics.first)))
+    else
+      $stdout.print '.'
+    end
+  end
+
+  private
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value)
+    if BenchmarkDriver::Result::ERROR.equal?(value)
+      return " %#{NAME_LENGTH}s" % 'ERROR'
+    elsif value == 0.0
+      return " %#{NAME_LENGTH}.3f" % 0.0
+    elsif value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    prefix = "%#{NAME_LENGTH}.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+end

data/lib/benchmark_driver/rbenv.rb

@@ -0,0 +1,25 @@
+require 'shellwords'
+
+module BenchmarkDriver
+  module Rbenv
+    # @param [String] version
+    def self.ruby_path(version)
+      path = `RBENV_VERSION='#{version}' rbenv which ruby`.rstrip
+      unless $?.success?
+        abort "Failed to execute 'rbenv which ruby'"
+      end
+      path
+    end
+
+    # @param [String] full_spec - "2.5.0", "2.5.0,--jit", "JIT::2.5.0,--jit", etc.
+    def self.parse_spec(full_spec)
+      name, spec = full_spec.split('::', 2)
+      spec ||= name # if `::` is not given, regard whole string as spec
+      version, *args = spec.shellsplit
+      BenchmarkDriver::Config::Executable.new(
+        name: name,
+        command: [BenchmarkDriver::Rbenv.ruby_path(version), *args],
+      )
+    end
+  end
+end

data/lib/benchmark_driver/repeater.rb

@@ -0,0 +1,52 @@
+module BenchmarkDriver
+  # Repeat calling block and return desired result: "best", "worst" or "average".
+  module Repeater
+    VALID_TYPES = %w[best worst average]
+
+    class << self
+      # `block.call` can return multiple objects, but the first one is used for sort.
+      # When `config.repeat_result == 'average'`, how to deal with rest objects is decided
+      # by `:rest_on_average` option.
+      def with_repeat(config:, larger_better:, rest_on_average: :first, &block)
+        values = config.repeat_count.times.map { block.call }
+
+        case config.repeat_result
+        when 'best'
+          best_result(values, larger_better)
+        when 'worst'
+          best_result(values, !larger_better)
+        when 'average'
+          average_result(values, rest_on_average)
+        else
+          raise "unexpected repeat_result #{config.repeat_result.inspect}"
+        end
+      end
+
+      private
+
+      def best_result(values, larger_better)
+        values.sort_by do |value, *|
+          larger_better ? value : -value
+        end.last
+      end
+
+      def average_result(values, rest_on_average)
+        unless values.first.is_a?(Array)
+          return values.inject(&:+) / values.size.to_f
+        end
+
+        case rest_on_average
+        when :first
+          rest = values.first[1..-1]
+          [values.map { |v| v[0] }.inject(&:+) / values.size.to_f, *rest]
+        when :average
+          values.first.size.times.map do |index|
+            values.map { |v| v[index] }.inject(&:+) / values.first.size.to_f
+          end
+        else
+          raise "unexpected rest_on_average #{rest_on_average.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/benchmark_driver/ruby_interface.rb

@@ -0,0 +1,83 @@
+module BenchmarkDriver
+  class RubyInterface
+    def self.run(**args, &block)
+      new(**args).tap { |x| block.call(x) }.run
+    end
+
+    # Build jobs and run. This is NOT interface for users.
+    def run
+      unless @executables.empty?
+        @config.executables = @executables
+      end
+
+      jobs = @jobs.flat_map do |job|
+        BenchmarkDriver::JobParser.parse({
+          type: @config.runner_type,
+          prelude: @prelude,
+          loop_count: @loop_count,
+        }.merge!(job))
+      end
+      BenchmarkDriver::Runner.run(jobs, config: @config)
+    end
+
+    #
+    # Config APIs from here
+    #
+
+    # @param [String,NilClass] output
+    # @param [String,NilClass] runner
+    def initialize(output: nil, runner: nil, repeat_count: 1)
+      @prelude = ''
+      @loop_count = nil
+      @jobs = []
+      @config = BenchmarkDriver::Config.new
+      @config.output_type = output.to_s if output
+      @config.runner_type = runner.to_s if runner
+      @config.repeat_count = Integer(repeat_count)
+      @executables = []
+    end
+
+    # @param [String] script
+    def prelude(script)
+      @prelude << "#{script}\n"
+    end
+
+    # @param [Integer] count
+    def loop_count(count)
+      @loop_count = count
+    end
+
+    # @param [String] name - Name shown on result output.
+    # @param [String,nil] script - Benchmarked script in String. If nil, name is considered as script too.
+    def report(name, script = nil)
+      if script.nil?
+        script = name
+      end
+      @jobs << { benchmark: [{ name: name, script: script }] }
+    end
+
+    def output(type)
+      @config.output_type = type
+    end
+
+    # Backward compatibility. This is actually default now.
+    def compare!
+      @config.output_type = 'compare'
+    end
+
+    def rbenv(*versions)
+      versions.each do |version|
+        @executables << BenchmarkDriver::Rbenv.parse_spec(version)
+      end
+    end
+
+    def executable(name:, command:)
+      raise ArgumentError, "`command' should be an Array" unless command.kind_of? Array
+      @executables << BenchmarkDriver::Config::Executable.new(name: name, command: command)
+    end
+
+    def verbose(level = 1)
+      @config.verbose = level
+    end
+  end
+end

data/lib/benchmark_driver/runner.rb

@@ -0,0 +1,103 @@
+module BenchmarkDriver
+  module Runner
+    require 'benchmark_driver/runner/command_stdout'
+    require 'benchmark_driver/runner/ips'
+    require 'benchmark_driver/runner/memory'
+    require 'benchmark_driver/runner/once'
+    require 'benchmark_driver/runner/recorded'
+    require 'benchmark_driver/runner/ruby_stdout'
+    require 'benchmark_driver/runner/time'
+  end
+
+  class << Runner
+    # Main function which is used by both CLI and `Benchmark.driver`.
+    # @param [Array<BenchmarkDriver::*::Job>] jobs
+    # @param [BenchmarkDriver::Config] config
+    def run(jobs, config:)
+      if config.verbose >= 1
+        config.executables.each do |exec|
+          $stdout.puts "#{exec.name}: #{IO.popen([*exec.command, '-v'], &:read)}"
+        end
+      end
+
+      runner_config = Config::RunnerConfig.new
+      runner_config.members.each do |member|
+        runner_config[member] = config[member]
+      end
+
+      jobs.group_by{ |j| j.respond_to?(:contexts) && j.contexts }.each do |contexts, contexts_jobs|
+        contexts_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
+          metrics_jobs.group_by(&:class).each do |klass, klass_jobs|
+            runner = runner_for(klass)
+            contexts = build_contexts(contexts, executables: config.executables)
+            output = Output.new(
+              type: config.output_type,
+              metrics: metrics,
+              jobs: klass_jobs.map { |job| BenchmarkDriver::Job.new(name: job.name) },
+              contexts: contexts,
+            )
+            with_clean_env do
+              runner.new(config: runner_config, output: output, contexts: contexts).run(klass_jobs)
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def build_contexts(contexts, executables:)
+      # If contexts are not specified, just use executables as contexts.
+      if !contexts.is_a?(Array) || contexts.empty?
+        return executables.map { |exec|
+          BenchmarkDriver::Context.new(name: exec.name, executable: exec)
+        }
+      end
+
+      with_executables, without_executables = contexts.partition { |context| context.name && context.executable }
+      with_executables + build_contexts_with_executables(without_executables, executables)
+    end
+
+    def build_contexts_with_executables(contexts, executables)
+      # Create direct product of contexts
+      contexts.product(executables).map do |context, executable|
+        name = context.name
+        if name.nil?
+          # Use the first gem name and version by default
+          name = context.gems.first.join(' ')
+
+          # Append Ruby executable name if it's matrix
+          if executables.size > 1
+            name = "#{name} (#{executable.name})"
+          end
+        end
+
+        BenchmarkDriver::Context.new(
+          name: name,
+          executable: executable,
+          gems: context.gems,
+          prelude: context.prelude,
+        )
+      end
+    end
+
+    # Dynamically find class (BenchmarkDriver::*::JobRunner) for plugin support
+    # @param [Class] klass - BenchmarkDriver::*::Job
+    # @return [Class]
+    def runner_for(klass)
+      unless match = klass.name.match(/\ABenchmarkDriver::Runner::(?<namespace>[^:]+)::Job\z/)
+        raise "Unexpected job class: #{klass}"
+      end
+      BenchmarkDriver.const_get("Runner::#{match[:namespace]}", false)
+    end
+
+    def with_clean_env(&block)
+      require 'bundler'
+      Bundler.with_clean_env do
+        block.call
+      end
+    rescue LoadError
+      block.call
+    end
+  end
+end