benchmark_driver 0.10.16 → 0.11.0

data/lib/benchmark_driver/metric.rb

@@ -0,0 +1,70 @@
+require 'benchmark_driver/struct'
+
+# All benchmark results should be expressed by this model.
+module BenchmarkDriver
+  # BenchmarkDriver returns benchmark results with the following nested Hash structure:
+  # {
+  #   BenchmarkDriver::Job => {
+  #     BenchmarkDriver::Context => {
+  #       BenchmarkDriver::Metric => Float
+  #     }
+  #   }
+  # }
+
+  # A kind of thing to be measured
+  Metric = ::BenchmarkDriver::Struct.new(
+    :name,          # @param [String] - Metric name or description like "Max Resident Set Size"
+    :unit,          # @param [String] - A unit like "MiB"
+    :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+    :worse_word,    # @param [String] - A label shown when the value is worse.
+    defaults: { larger_better: true, worse_word: 'slower' },
+  )
+
+  # Benchmark conditions used to measure a metric
+  Context = ::BenchmarkDriver::Struct.new(
+    :name,        # @param [String] - Name of the context
+    :executable,  # @param [BenchmarkDriver::Config::Executable] - Measured Ruby executable
+    :gems,        # @param [Hash{ String => String,nil }] - Gem -> version pairs used for the benchmark
+    :prelude,     # @param [String,nil] - Context specific setup script (optional)
+    :duration,    # @param [Float,nil] - Time taken to run the benchmark job (optional)
+    :loop_count,  # @param [Integer,nil] - Times to run the benchmark job (optional)
+    :environment, # @param [Hash] - Any other key -> value pairs to express the benchmark context
+    defaults: { gems: {}, environment: {} },
+  )
+
+  # Holding identifier of measured workload
+  Job = ::BenchmarkDriver::Struct.new(
+    :name, # @param [String] - Name of the benchmark task
+  )
+
+  #=[RubyBench mapping]=================================|
+  #
+  # BenchmarkRun:
+  #   result              -> { context.name => value }  | { "default"=>"44.666666666666664", "default_jit"=>"59.333333333333336" }
+  #   environment         -> context                    | "---\nRuby version: 'ruby 2.6.0dev (2018-05-14 trunk 63417) [x86_64-linux]\n\n'\nChecksum: '59662'\n"
+  #   initiator           -> (not supported)            | #<Commit sha1: "6f0de6ed9...", message: "error.c: check redefined ...", url: "https://github.com/tgxworld/ruby/commit/6f0de6ed98...", repo_id: 6>
+  #
+  #   BenchmarkType:
+  #     category          -> job.name                   | "app_erb", "Optcarrot Lan_Master.nes"
+  #     script_url        -> (not supported)            | "https://raw.githubusercontent.com/mame/optcarrot/master/lib/optcarrot/nes.rb"
+  #     repo              -> (not supported)            | #<Repo name: "ruby", url: "https://github.com/tgxworld/ruby">
+  #     repo.organization -> (not supported)            | #<Organization name: "ruby", url: "https://github.com/tgxworld/">
+  #
+  #   BenchmarkResultType:
+  #     name              -> metric.name                | "Number of frames"
+  #     unit              -> metric.unit                | "fps"
+  #
+  #=====================================================|
+
+  #----
+  # legacy
+
+  module Metrics
+    Type = ::BenchmarkDriver::Struct.new(
+      :unit,          # @param [String] - A label of unit for the value.
+      :larger_better, # @param [TrueClass,FalseClass] - If true, larger value is preferred when measured multiple times.
+      :worse_word,    # @param [String] - A label shown when the value is worse.
+      defaults: { larger_better: true, worse_word: 'slower' },
+    )
+  end
+end

data/lib/benchmark_driver/output.rb

@@ -1,29 +1,83 @@
 module BenchmarkDriver
-  module Output
+  # BenchmarkDriver::Runner::* --> BenchmarkDriver::Output --> BenchmarkDriver::Output::*
+  #
+  # This is interface between runner plugin and output plugin, so that they can be loosely
+  # coupled and to simplify implementation of both runner and output.
+  #
+  # Runner should call its interface in the following manner:
+  #   metrics=
+  #   with_warmup
+  #     with_job(name:)
+  #       with_context(name:, executable:, duration: nil, loop_count: nil)
+  #         report(value:, metric:)
+  #   with_benchmark
+  #     with_job(name:)
+  #       with_context(name:, executable:, duration: nil, loop_count: nil)
+  #         report(value:, metric:)
+  class Output
     require 'benchmark_driver/output/compare'
     require 'benchmark_driver/output/markdown'
     require 'benchmark_driver/output/record'
     require 'benchmark_driver/output/simple'
-  end
 
-  class << Output
     # BenchmarkDriver::Output is pluggable.
     # Create `BenchmarkDriver::Output::Foo` as benchmark_dirver-output-foo.gem and specify `-o foo`.
     #
     # @param [String] type
-    def find(type)
+    # @param [Array<String>] job_names
+    # @param [Array<String>] context_names
+    def initialize(type:, job_names:, context_names:)
       if type.include?(':')
         raise ArgumentError.new("Output type '#{type}' cannot contain ':'")
       end
 
       require "benchmark_driver/output/#{type}" # for plugin
-      ::BenchmarkDriver::Output.const_get(camelize(type), false)
+      camelized = type.split('_').map(&:capitalize).join
+
+      @output = ::BenchmarkDriver::Output.const_get(camelized, false).new(
+        job_names: job_names,
+        context_names: context_names,
+      )
+    end
+
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    def metrics=(metrics)
+      @output.metrics = metrics
     end
 
-    private
+    def with_warmup(&block)
+      @output.with_warmup(&block)
+    end
+
+    def with_benchmark(&block)
+      @output.with_benchmark(&block)
+    end
+
+    # @param [String] name
+    def with_job(name:, &block)
+      job = BenchmarkDriver::Job.new(name: name)
+      @output.with_job(job) do
+        block.call
+      end
+    end
+
+    # @param [String] name
+    # @param [BenchmarkDriver::Config::Executable] executable
+    # @param [Float] duration
+    # @param [Integer] loop_count
+    def with_context(name:, executable:, duration: nil, loop_count: nil, environment: {}, &block)
+      context = BenchmarkDriver::Context.new(
+        name: name, executable: executable, duration: duration, loop_count: loop_count, environment: environment,
+      )
+      @output.with_context(context) do
+        block.call
+      end
+    end
 
-    def camelize(str)
-      str.split('_').map(&:capitalize).join
+    # @param [Float] value
+    # @param [BenchmarkDriver::Metric] metic
+    def report(value:, metric:)
+      @output.report(value: value, metric: metric)
     end
   end
 end

data/lib/benchmark_driver/output/compare.rb

@@ -2,14 +2,15 @@
 class BenchmarkDriver::Output::Compare
   NAME_LENGTH = 20
 
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  attr_writer :metrics_type
-
-  # @param [Array<BenchmarkDriver::*::Job>] jobs
-  # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  def initialize(jobs:, executables:)
-    @jobs = jobs
-    @executables = executables
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  attr_writer :metrics
+
+  # @param [Array<String>] job_names
+  # @param [Array<String>] context_names
+  def initialize(job_names:, context_names:)
+    @job_names = job_names
+    @context_names = context_names
+    @name_length = [job_names.map(&:length).max, NAME_LENGTH].max
   end
 
   def with_warmup(&block)
@@ -21,14 +22,16 @@ class BenchmarkDriver::Output::Compare
   end
 
   def with_benchmark(&block)
-    @metrics_by_job = Hash.new { |h, k| h[k] = [] }
+    @job_context_values = Hash.new do |h1, k1|
+      h1[k1] = Hash.new { |h2, k2| h2[k2] = [] }
+    end
 
     without_stdout_buffering do
       $stdout.puts 'Calculating -------------------------------------'
-      if @executables.size > 1
-        $stdout.print(' ' * NAME_LENGTH)
-        @executables.each do |executable|
-          $stdout.print(' %10s ' % executable.name)
+      if @context_names.size > 1
+        $stdout.print(' ' * @name_length)
+        @context_names.each do |context_name|
+          $stdout.print(' %10s ' % context_name)
         end
         $stdout.puts
       end
@@ -36,28 +39,30 @@ class BenchmarkDriver::Output::Compare
       block.call
     end
   ensure
-    if @executables.size > 1
+    if @context_names.size > 1
       compare_executables
-    elsif @jobs.size > 1
+    elsif @job_names.size > 1
       compare_jobs
     end
   end
 
-  # @param [BenchmarkDriver::*::Job] job
+  # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
-    if job.name.length > NAME_LENGTH
-      $stdout.puts(job.name)
+    name = job.name
+    if name.length > @name_length
+      $stdout.puts(name)
     else
-      $stdout.print("%#{NAME_LENGTH}s" % job.name)
+      $stdout.print("%#{@name_length}s" % name)
     end
-    @current_job = job
-    @job_metrics = []
+    @job = name
+    @job_contexts = []
     block.call
   ensure
-    $stdout.print(@metrics_type.unit)
-    if job.respond_to?(:loop_count) && job.loop_count
-      $stdout.print(" - #{humanize(job.loop_count)} times")
-      if @job_metrics.all? { |metrics| metrics.duration }
+    $stdout.print(@metrics.first.unit)
+    loop_count = @job_contexts.first.loop_count
+    if loop_count && @job_contexts.all? { |c| c.loop_count == loop_count }
+      $stdout.print(" - #{humanize(loop_count)} times")
+      if @job_contexts.all? { |context| !context.duration.nil? }
         $stdout.print(" in")
         show_durations
       end
@@ -65,28 +70,35 @@ class BenchmarkDriver::Output::Compare
     $stdout.puts
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
-  def report(metrics)
-    if defined?(@metrics_by_job)
-      @metrics_by_job[@current_job] << metrics
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    @context = context
+    @job_contexts << context
+    block.call
+  end
+
+  # @param [Float] value
+  # @param [BenchmarkDriver::Metric] metic
+  def report(value:, metric:)
+    if defined?(@job_context_values)
+      @job_context_values[@job][@context] << value
     end
 
-    @job_metrics << metrics
-    $stdout.print("#{humanize(metrics.value, [10, metrics.executable.name.length].max)} ")
+    $stdout.print("#{humanize(value, [10, @context.name.length].max)} ")
   end
 
   private
 
   def show_durations
-    @job_metrics.each do |metrics|
-      $stdout.print(' %3.6fs' % metrics.duration)
+    @job_contexts.each do |context|
+      $stdout.print(' %3.6fs' % context.duration)
     end
 
     # Show pretty seconds / clocks too. As it takes long width, it's shown only with a single executable.
-    if @job_metrics.size == 1
-      metrics = @job_metrics.first
-      sec = metrics.duration
-      iter = @current_job.loop_count
+    if @job_contexts.size == 1
+      context = @job_contexts.first
+      sec = context.duration
+      iter = context.loop_count
       if File.exist?('/proc/cpuinfo') && (clks = estimate_clock(sec, iter)) < 1_000
         $stdout.print(" (#{pretty_sec(sec, iter)}/i, #{clks}clocks/i)")
       else
@@ -148,16 +160,20 @@ class BenchmarkDriver::Output::Compare
 
   def compare_jobs
     $stdout.puts "\nComparison:"
-    results = @metrics_by_job.map { |job, metrics| Result.new(job: job, metrics: metrics.first) }
+    results = @job_context_values.flat_map do |job, context_values|
+      context_values.map { |context, values| Result.new(job: job, value: values.first, executable: context.executable) }
+    end
     show_results(results, show_executable: false)
   end
 
   def compare_executables
     $stdout.puts "\nComparison:"
 
-    @metrics_by_job.each do |job, metrics|
-      $stdout.puts("%#{NAME_LENGTH + 2 + 11}s" % job.name)
-      results = metrics.map { |metrics| Result.new(job: job, metrics: metrics) }
+    @job_context_values.each do |job, context_values|
+      $stdout.puts("%#{@name_length + 2 + 11}s" % job)
+      results = context_values.flat_map do |context, values|
+        values.map { |value| Result.new(job: job, value: value, executable: context.executable) }
+      end
       show_results(results, show_executable: true)
     end
   end
@@ -166,32 +182,32 @@ class BenchmarkDriver::Output::Compare
   # @param [TrueClass,FalseClass] show_executable
   def show_results(results, show_executable:)
     results = results.sort_by do |result|
-      if @metrics_type.larger_better
-        -result.metrics.value
+      if @metrics.first.larger_better
+        -result.value
       else
-        result.metrics.value
+        result.value
       end
     end
 
     first = results.first
     results.each do |result|
       if result != first
-        if @metrics_type.larger_better
-          ratio = (first.metrics.value / result.metrics.value)
+        if @metrics.first.larger_better
+          ratio = (first.value / result.value)
         else
-          ratio = (result.metrics.value / first.metrics.value)
+          ratio = (result.value / first.value)
         end
-        slower = "- %.2fx  #{@metrics_type.worse_word}" % ratio
+        slower = "- %.2fx  #{@metrics.first.worse_word}" % ratio
       end
       if show_executable
-        name = result.metrics.executable.name
+        name = result.executable.name
       else
-        name = result.job.name
+        name = result.job
       end
-      $stdout.puts("%#{NAME_LENGTH}s: %11.1f %s #{slower}" % [name, result.metrics.value, @metrics_type.unit])
+      $stdout.puts("%#{@name_length}s: %11.1f %s #{slower}" % [name, result.value, @metrics.first.unit])
     end
     $stdout.puts
   end
 
-  Result = ::BenchmarkDriver::Struct.new(:job, :metrics)
+  Result = ::BenchmarkDriver::Struct.new(:job, :value, :executable)
 end

data/lib/benchmark_driver/output/markdown.rb

@@ -1,15 +1,14 @@
 class BenchmarkDriver::Output::Markdown
   NAME_LENGTH = 8
 
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  attr_writer :metrics_type
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  attr_writer :metrics
 
-  # @param [Array<BenchmarkDriver::*::Job>] jobs
-  # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  def initialize(jobs:, executables:)
-    @jobs = jobs
-    @executables = executables
-    @name_length = jobs.map { |j| j.name.size }.max
+  # @param [Array<String>] job_names
+  # @param [Array<String>] context_names
+  def initialize(job_names:, context_names:)
+    @context_names = context_names
+    @name_length = job_names.map(&:size).max
   end
 
   def with_warmup(&block)
@@ -25,18 +24,18 @@ class BenchmarkDriver::Output::Markdown
     @with_benchmark = true
     without_stdout_buffering do
       # Show header
-      $stdout.puts "# benchmark results (#{@metrics_type.unit})\n\n"
+      $stdout.puts "# #{@metrics.first.name} (#{@metrics.first.unit})\n\n"
 
       # Show executable names
       $stdout.print("|#{' ' * @name_length}  ")
-      @executables.each do |executable|
-        $stdout.print("|%#{NAME_LENGTH}s" % executable.name) # same size as humanize
+      @context_names.each do |context_name|
+        $stdout.print("|%#{NAME_LENGTH}s" % context_name) # same size as humanize
       end
       $stdout.puts('|')
 
       # Show header separator
       $stdout.print("|:#{'-' * (@name_length - 1)}--")
-      @executables.each do |executable|
+      @context_names.each do |context_name|
         $stdout.print("|:#{'-' * (NAME_LENGTH - 1)}") # same size as humanize
       end
       $stdout.puts('|')
@@ -47,7 +46,7 @@ class BenchmarkDriver::Output::Markdown
     @with_benchmark = false
   end
 
-  # @param [BenchmarkDriver::*::Job] job
+  # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
     if @with_benchmark
       $stdout.print("|%-#{@name_length}s  " % job.name)
@@ -59,10 +58,16 @@ class BenchmarkDriver::Output::Markdown
     end
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
-  def report(metrics)
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    block.call
+  end
+
+  # @param [Floa] value
+  # @param [BenchmarkDriver::Metric] metic
+  def report(value:, metric:)
     if @with_benchmark
-      $stdout.print("|%#{NAME_LENGTH}s" % humanize(metrics.value))
+      $stdout.print("|%#{NAME_LENGTH}s" % humanize(value))
     else
       $stdout.print '.'
     end

data/lib/benchmark_driver/output/record.rb

@@ -1,14 +1,16 @@
 class BenchmarkDriver::Output::Record
-  # @param [BenchmarkDriver::Metrics::Type] metrics_type
-  attr_writer :metrics_type
-
-  # @param [Array<BenchmarkDriver::*::Job>] jobs
-  # @param [Array<BenchmarkDriver::Config::Executable>] executables
-  def initialize(jobs:, executables:)
-    @jobs = jobs
-    @executables = executables
-    @metrics_by_job = Hash.new do |h1, k1|
-      h1[k1] = Hash.new { |h2, k2| h2[k2] = [] }
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  attr_writer :metrics
+
+  # @param [Array<String>] job_names
+  # @param [Array<String>] context_names
+  def initialize(job_names:, context_names:)
+    @job_warmup_context_metric_value = Hash.new do |h1, k1|
+      h1[k1] = Hash.new do |h2, k2|
+        h2[k2] = Hash.new do |h3, k3|
+          h3[k3] = {}
+        end
+      end
     end
   end
 
@@ -29,20 +31,23 @@ class BenchmarkDriver::Output::Record
     save_record
   end
 
-  # @param [BenchmarkDriver::*::Job] job
+  # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
-    @current_job = job
+    @job = job.name
     block.call
   end
 
-  # @param [BenchmarkDriver::Metrics] metrics
-  def report(metrics)
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    @context = context
+    block.call
+  end
+
+  # @param [Float] value
+  # @param [BenchmarkDriver::Metric] metic
+  def report(value:, metric:)
     $stdout.print '.'
-    if @with_benchmark
-      @metrics_by_job[@current_job][:benchmark] << metrics
-    else
-      @metrics_by_job[@current_job][:warmup] << metrics
-    end
+    @job_warmup_context_metric_value[@job][!@with_benchmark][@context][metric] = value
   end
 
   private
@@ -51,8 +56,8 @@ class BenchmarkDriver::Output::Record
     jobs = @benchmark_metrics
     yaml = {
       'type' => 'recorded',
-      'metrics_by_job' => @metrics_by_job,
-      'metrics_type' => @metrics_type,
+      'job_warmup_context_metric_value' => @job_warmup_context_metric_value,
+      'metrics' => @metrics,
     }.to_yaml
     File.write('benchmark_driver.record.yml', yaml)
   end