benchmark_driver_monotonic_raw 0.14.13

data/lib/benchmark_driver/default_job.rb

@@ -0,0 +1,29 @@
+require 'benchmark_driver/struct'
+
+module BenchmarkDriver
+  DefaultJob = ::BenchmarkDriver::Struct.new(
+    :name,       # @param [String] name - This is mandatory for all runner
+    :metrics,    # @param [Array<BenchmarkDriver::Metric>] - This is mandatory for all runner too, set by job parser.
+    :contexts,   # @param [Array<BenchmarkDriver::Context>] - This is optional parameter for runners.
+    :script,     # @param [String] benchmark
+    :prelude,    # @param [String,nil] prelude (optional)
+    :teardown,   # @param [String,nil] after (optional)
+    :loop_count, # @param [Integer,nil] loop_count (optional)
+    :required_ruby_version, # @param [String,nil] required_ruby_version (optional)
+    defaults: { prelude: '', teardown: '' },
+  ) do
+    def runnable_contexts(contexts)
+      if required_ruby_version
+        contexts.select do |context|
+          Gem::Version.new(context.executable.version) >= Gem::Version.new(required_ruby_version)
+        end.tap do |result|
+          if result.empty?
+            raise "No Ruby executables conforming required_ruby_version (#{required_ruby_version}) are specified"
+          end
+        end
+      else
+        contexts
+      end
+    end
+  end
+end

data/lib/benchmark_driver/default_job_parser.rb

@@ -0,0 +1,91 @@
+module BenchmarkDriver
+  module DefaultJobParser
+    # Build default JobParser for given job klass
+    def self.for(klass:, metrics:)
+      Module.new.tap do |parser|
+        class << parser
+          include DefaultJobParser
+        end
+        parser.define_singleton_method(:job_class) { klass }
+        parser.define_singleton_method(:job_metrics) { metrics }
+      end
+    end
+
+    # This method is dynamically called by `BenchmarkDriver::JobParser.parse`
+    # @param [String] prelude
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    # @param [String] teardown
+    # @param [Integer] loop_count
+    # @param [String] required_ruby_version
+    # @return [Array<BenchmarkDriver::Default::Job>]
+    def parse(contexts: [], prelude: nil, benchmark:, teardown: nil, loop_count: nil, required_ruby_version: nil)
+      parse_benchmark(benchmark).each do |job|
+        job.contexts = parse_contexts(contexts)
+        job.metrics = job_metrics
+        job.prelude.prepend("#{prelude}\n") if prelude
+        job.teardown.prepend("#{teardown}\n") if teardown
+        job.loop_count ||= loop_count
+        job.required_ruby_version ||= required_ruby_version
+      end.each(&:freeze)
+    end
+
+    private
+
+    # @param [String,Array<String,Hash{ Symbol => String }>,Hash{ Symbol => String }] benchmark
+    def parse_benchmark(benchmark)
+      case benchmark
+      when String
+        [parse_job(benchmark)]
+      when Array
+        benchmark.map { |b| parse_job(b) }
+      when Hash
+        benchmark.map do |key, value|
+          job_class.new(name: key.to_s, script: value)
+        end
+      else
+        raise ArgumentError.new("benchmark must be String, Array or Hash, but got: #{benchmark.inspect}")
+      end
+    end
+
+    # @param [String,Hash{ Symbol => String }>] bench
+    def parse_job(benchmark)
+      case benchmark
+      when String
+        job_class.new(name: benchmark, script: benchmark)
+      when Hash
+        parse_job_hash(benchmark)
+      else
+        raise ArgumentError.new("Expected String or Hash in element of benchmark, but got: #{benchmark.inspect}")
+      end
+    end
+
+    def parse_job_hash(name: nil, prelude: '', script:, teardown: '', loop_count: nil, required_ruby_version: nil)
+      name ||= script
+      job_class.new(name: name, prelude: prelude, script: script, teardown: teardown, loop_count: loop_count, required_ruby_version: required_ruby_version)
+    end
+
+    def job_class
+      raise NotImplementedError # override this
+    end
+
+    def parse_contexts(contexts)
+      if contexts.is_a?(Array)
+        contexts.map { |context| parse_context(context) }
+      else
+        raise ArgumentError.new("contexts must be Array, but got: #{contexts.inspect}")
+      end
+    end
+
+    def parse_context(name: nil, prelude: '', gems: {}, require: true)
+      gems.each do |gem, version|
+        prelude = "gem '#{gem}', '#{version}'\n#{("require '#{gem}'\n" if require)}#{prelude}"
+      end
+
+      BenchmarkDriver::Context.new(
+        name: name,
+        gems: gems,
+        prelude: prelude,
+      )
+    end
+  end
+end

data/lib/benchmark_driver/job_parser.rb

@@ -0,0 +1,55 @@
+require 'benchmark_driver/runner'
+
+module BenchmarkDriver
+  class << JobParser = Module.new
+    # @param [Hash] config
+    # @param [Hash] default_params - Special default values not written in job definition
+    def parse(config, default_params: {})
+      config = symbolize_keys(config)
+      type = config.fetch(:type)
+      if !type.is_a?(String)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
+      elsif !type.match(/\A[A-Za-z0-9_]+\z/)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
+      end
+      config.delete(:type)
+
+      # Dynamic dispatch for plugin support
+      require "benchmark_driver/runner/#{type}"
+      job = ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(config)
+      default_params.each do |key, value|
+        if job.respond_to?(key) && job.respond_to?("#{key}=") && job.public_send(key).nil?
+          job.public_send("#{key}=", value)
+        end
+      end
+      job
+    end
+
+    private
+
+    def camelize(str)
+      str.split('_').map(&:capitalize).join
+    end
+
+    # @param [Object] config
+    def symbolize_keys(config)
+      case config
+      when Hash
+        config.dup.tap do |hash|
+          hash.keys.each do |key|
+            case key
+            when String, Symbol
+              hash[key.to_sym] = symbolize_keys(hash.delete(key))
+            else # Struct
+              hash[key] = symbolize_keys(hash.delete(key))
+            end
+          end
+        end
+      when Array
+        config.map { |c| symbolize_keys(c) }
+      else
+        config
+      end
+    end
+  end
+end

data/lib/benchmark_driver/metric.rb

@@ -0,0 +1,79 @@
+require 'benchmark_driver/struct'
+
+# All benchmark results should be expressed by this model.
+module BenchmarkDriver
+  # BenchmarkDriver returns benchmark results with the following nested Hash structure:
+  # {
+  #   #<BenchmarkDriver::Job> => {
+  #     #<BenchmarkDriver::Context> => #<BenchmarkDriver::Result
+  #       metrics: {
+  #         #<BenchmarkDriver::Metric> => Float
+  #       }
+  #     >
+  #   }
+  # }
+
+  # Holding identifier of measured workload
+  Job = ::BenchmarkDriver::Struct.new(
+    :name, # @param [String] - Name of the benchmark task
+  )
+
+  # Benchmark conditions that can be known before running benchmark
+  Context = ::BenchmarkDriver::Struct.new(
+    :name,        # @param [String] - Name of the context
+    :executable,  # @param [BenchmarkDriver::Config::Executable] - Measured Ruby executable
+    :gems,        # @param [Hash{ String => String,nil }] - Gem -> version pairs used for the benchmark
+    :prelude,     # @param [String,nil] - Context specific setup script (optional)
+    defaults: { prelude: '', gems: {} },
+  )
+
+  # Everything that can be known after running benchmark
+  Result = ::BenchmarkDriver::Struct.new(
+    :values,      # @param [Hash{ BenchmarkDriver::Metric => Float }] - Main benchmark results
+    :duration,    # @param [Float,nil] - Time taken to run the benchmark job (optional)
+    :loop_count,  # @param [Integer,nil] - Times to run the benchmark job (optional)
+    :environment, # @param [Hash] - Any other key -> value pairs to express the benchmark context
+    defaults: { environment: {} },
+  )
+  Result::ERROR = 0
+
+  # A kind of thing to be measured
+  Metric = ::BenchmarkDriver::Struct.new(
+    :name,          # @param [String] - Metric name or description like "Max Resident Set Size"
+    :unit,          # @param [String] - A unit like "MiB"
+    :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+    :worse_word,    # @param [String] - A label shown when the value is worse.
+    defaults: { larger_better: true, worse_word: 'slower' },
+  )
+
+  #=[RubyBench mapping]=======================================|
+  #
+  # BenchmarkRun:
+  #   result              -> { context.name => result.value } | { "default"=>"44.666666666666664", "default_jit"=>"59.333333333333336" }
+  #   environment         -> result.environment               | "---\nRuby version: 'ruby 2.6.0dev (2018-05-14 trunk 63417) [x86_64-linux]\n\n'\nChecksum: '59662'\n"
+  #   initiator           -> (not supported)                  | #<Commit sha1: "6f0de6ed9...", message: "error.c: check redefined ...", url: "https://github.com/tgxworld/ruby/commit/6f0de6ed98...", repo_id: 6>
+  #
+  #   BenchmarkType:
+  #     category          -> job.name                         | "app_erb", "Optcarrot Lan_Master.nes"
+  #     script_url        -> (not supported)                  | "https://raw.githubusercontent.com/mame/optcarrot/master/lib/optcarrot/nes.rb"
+  #     repo              -> (not supported)                  | #<Repo name: "ruby", url: "https://github.com/tgxworld/ruby">
+  #     repo.organization -> (not supported)                  | #<Organization name: "ruby", url: "https://github.com/tgxworld/">
+  #
+  #   BenchmarkResultType:
+  #     name              -> metric.name                      | "Number of frames"
+  #     unit              -> metric.unit                      | "fps"
+  #
+  #===========================================================|
+
+  #----
+  # legacy
+
+  module Metrics
+    Type = ::BenchmarkDriver::Struct.new(
+      :unit,          # @param [String] - A label of unit for the value.
+      :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+      :worse_word,    # @param [String] - A label shown when the value is worse.
+      defaults: { larger_better: true, worse_word: 'slower' },
+    )
+  end
+end

data/lib/benchmark_driver/output.rb

@@ -0,0 +1,88 @@
+module BenchmarkDriver
+  # BenchmarkDriver::Runner::* --> BenchmarkDriver::Output --> BenchmarkDriver::Output::*
+  #
+  # This is interface between runner plugin and output plugin, so that they can be loosely
+  # coupled and to simplify implementation of both runner and output.
+  #
+  # Runner should call its interface in the following manner:
+  #   metrics=
+  #   with_warmup
+  #     with_job(name:)
+  #       with_context(name:, executable:, gems:)
+  #         report(values:, duration: nil, loop_count: nil, environment: {})
+  #   with_benchmark
+  #     with_job(name:)
+  #       with_context(name:, executable:, gems:)
+  #         report(values:, duration: nil, loop_count: nil, environment: {})
+  class Output
+    require 'benchmark_driver/output/compare'
+    require 'benchmark_driver/output/markdown'
+    require 'benchmark_driver/output/record'
+    require 'benchmark_driver/output/simple'
+
+    # BenchmarkDriver::Output is pluggable.
+    # Create `BenchmarkDriver::Output::Foo` as benchmark_dirver-output-foo.gem and specify `-o foo`.
+    #
+    # @param [String] type
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    # @param [Array<BenchmarkDriver::Job>] jobs
+    # @param [Array<BenchmarkDriver::Context>] contexts
+    def initialize(type:, metrics:, jobs:, contexts:)
+      if type.include?(':')
+        raise ArgumentError.new("Output type '#{type}' cannot contain ':'")
+      end
+
+      require "benchmark_driver/output/#{type}" # for plugin
+      camelized = type.split('_').map(&:capitalize).join
+
+      @output = ::BenchmarkDriver::Output.const_get(camelized, false).new(
+        metrics: metrics,
+        jobs: jobs,
+        contexts: contexts,
+      )
+    end
+
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    def metrics=(metrics)
+      @output.metrics = metrics
+    end
+
+    def with_warmup(&block)
+      @output.with_warmup(&block)
+    end
+
+    def with_benchmark(&block)
+      @output.with_benchmark(&block)
+    end
+
+    # @param [String] name
+    def with_job(name:, &block)
+      job = BenchmarkDriver::Job.new(name: name)
+      @output.with_job(job) do
+        block.call
+      end
+    end
+
+    # @param [String] name
+    # @param [BenchmarkDriver::Config::Executable] executable
+    # @param [Hash{ String => String}] gems
+    def with_context(name:, executable:, gems: {}, prelude: '', &block)
+      context = BenchmarkDriver::Context.new(name: name, executable: executable, gems: gems, prelude: prelude)
+      @output.with_context(context) do
+        block.call
+      end
+    end
+
+    # @param [Float] value
+    # @param [BenchmarkDriver::Metric] metic
+    def report(values:, duration: nil, loop_count: nil, environment: {})
+      result = BenchmarkDriver::Result.new(
+        values: values,
+        duration: duration,
+        loop_count: loop_count,
+        environment: environment,
+      )
+      @output.report(result)
+    end
+  end
+end

data/lib/benchmark_driver/output/compare.rb

@@ -0,0 +1,216 @@
+# Compare output like benchmark-ips
+class BenchmarkDriver::Output::Compare
+  NAME_LENGTH = 20
+
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @job_names = jobs.map(&:name)
+    @context_names = contexts.map(&:name)
+    @name_length = [@job_names.map(&:length).max, NAME_LENGTH].max
+  end
+
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.puts 'Warming up --------------------------------------'
+      # TODO: show exec name if it has multiple ones
+      block.call
+    end
+  end
+
+  def with_benchmark(&block)
+    @job_context_result = Hash.new do |hash, job|
+      hash[job] = {}
+    end
+
+    result = without_stdout_buffering do
+      $stdout.puts 'Calculating -------------------------------------'
+      if @context_names.size > 1
+        $stdout.print(' ' * @name_length)
+        @context_names.each do |context_name|
+          $stdout.print(' %10s ' % context_name)
+        end
+        $stdout.puts
+      end
+
+      block.call
+    end
+    if @context_names.size > 1
+      compare_executables
+    elsif @job_names.size > 1
+      compare_jobs
+    end
+    result
+  end
+
+  # @param [BenchmarkDriver::Job] job
+  def with_job(job, &block)
+    name = job.name
+    if name.length > @name_length
+      $stdout.puts(name)
+    else
+      $stdout.print("%#{@name_length}s" % name)
+    end
+    @job = name
+    @job_results = []
+    @job_contexts = []
+    result = block.call
+    $stdout.print(@metrics.first.unit)
+    loop_count = @job_results.first.loop_count
+    if loop_count && @job_results.all? { |r| r.loop_count == loop_count }
+      $stdout.print(" - #{humanize(loop_count)} times")
+      if @job_results.all? { |job_result| !job_result.duration.nil? }
+        $stdout.print(" in")
+        show_durations
+      end
+    end
+    $stdout.puts
+    result
+  end
+
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    @context = context
+    @job_contexts << context
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    @job_results << result
+    if defined?(@job_context_result)
+      @job_context_result[@job][@context] = result
+    end
+
+    $stdout.print("#{humanize(result.values.values.first, [10, @context.name.length].max)} ")
+  end
+
+  private
+
+  def show_durations
+    @job_results.each do |result|
+      $stdout.print(' %3.6fs' % result.duration)
+    end
+
+    # Show pretty seconds / clocks too. As it takes long width, it's shown only with a single executable.
+    if @job_results.size == 1
+      result = @job_results.first
+      sec = result.duration
+      iter = result.loop_count
+      if File.exist?('/proc/cpuinfo') && (clks = estimate_clock(sec, iter)) < 1_000
+        $stdout.print(" (#{pretty_sec(sec, iter)}/i, #{clks}clocks/i)")
+      else
+        $stdout.print(" (#{pretty_sec(sec, iter)}/i)")
+      end
+    end
+  end
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+
+  def humanize(value, width = 10)
+    if BenchmarkDriver::Result::ERROR.equal?(value)
+      return " %#{width}s" % 'ERROR'
+    elsif value == 0.0
+      return " %#{width}.3f" % 0.0
+    elsif value < 0
+      raise ArgumentError.new("Negative value: #{value.inspect}")
+    end
+
+    scale = (Math.log10(value) / 3).to_i
+    return "%#{width}s" % value.to_s if scale < 0 # like 1.23e-04
+
+    prefix = "%#{width}.3f" % (value.to_f / (1000 ** scale))
+    suffix =
+      case scale
+      when 1; 'k'
+      when 2; 'M'
+      when 3; 'G'
+      when 4; 'T'
+      when 5; 'Q'
+      else # < 1000 or > 10^15, no scale or suffix
+        return " #{prefix}"
+      end
+    "#{prefix}#{suffix}"
+  end
+
+  def pretty_sec(sec, iter)
+    r = Rational(sec, iter)
+    case
+    when r >= 1
+      "#{'%3.2f' % r.to_f}s"
+    when r >= 1/1000r
+      "#{'%3.2f' % (r * 1_000).to_f}ms"
+    when r >= 1/1000_000r
+      "#{'%3.2f' % (r * 1_000_000).to_f}μs"
+    else
+      "#{'%3.2f' % (r * 1_000_000_000).to_f}ns"
+    end
+  end
+
+  def estimate_clock sec, iter
+    hz = File.read('/proc/cpuinfo').scan(/cpu MHz\s+:\s+([\d\.]+)/){|(f)| break hz = Rational(f.to_f) * 1_000_000}
+    r = Rational(sec, iter)
+    Integer(r/(1/hz))
+  end
+
+  def compare_jobs
+    $stdout.puts "\nComparison:"
+    results = @job_context_result.flat_map do |job, context_result|
+      context_result.map { |context, result| Result.new(job: job, value: result.values.values.first, context: context) }
+    end
+    show_results(results, show_context: false)
+  end
+
+  def compare_executables
+    $stdout.puts "\nComparison:"
+
+    @job_context_result.each do |job, context_result|
+      $stdout.puts("%#{@name_length + 2 + 11}s" % job)
+      results = context_result.flat_map do |context, result|
+        result.values.values.map { |value| Result.new(job: job, value: value, context: context) }
+      end
+      show_results(results, show_context: true)
+    end
+  end
+
+  # @param [Array<BenchmarkDriver::Output::Compare::Result>] results
+  # @param [TrueClass,FalseClass] show_context
+  def show_results(results, show_context:)
+    results = results.sort_by do |result|
+      if @metrics.first.larger_better
+        -result.value
+      else
+        result.value
+      end
+    end
+
+    first = results.first
+    results.each do |result|
+      if result != first
+        if @metrics.first.larger_better
+          ratio = (first.value / result.value)
+        else
+          ratio = (result.value / first.value)
+        end
+        slower = "- %.2fx  #{@metrics.first.worse_word}" % ratio
+      end
+      if show_context
+        name = result.context.name
+      else
+        name = result.job
+      end
+      $stdout.puts("%#{@name_length}s: %11.1f %s #{slower}" % [name, result.value, @metrics.first.unit])
+    end
+    $stdout.puts
+  end
+
+  Result = ::BenchmarkDriver::Struct.new(:job, :value, :context)
+end