benchmark_driver 0.11.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16b3e45ce8476d4acf4e3528d04079f1346b6436
-  data.tar.gz: 0e4a7a94f47ba9bb596ba96904712db4c6935032
+  metadata.gz: 606b300f9ffeb798a6ac330be1e97795898afa7c
+  data.tar.gz: fb3cfa3a44eff01b189635c8ebaffcb73737567d
 SHA512:
-  metadata.gz: 75b834584e0fc996dda837d879af768c6bab9acba07fb53f284103ec73aafd53b8b5cc82001b9067a7d0403420c97fa30f59146bcff9c3341c89cf4c003e915e
-  data.tar.gz: cafd73b0bdaeb3b8635a0cbc847eaf9d3b6969d80369093004db615618cab41b6e004640f81a58bd8d38389243fc4061c230473ed696b7524bc15d23bbe6d87d
+  metadata.gz: 8401a061574125ed5af0be54293fb2936bf8f856abb23c242232dff213770a008726839742353194d790ee6e883a00cfc1c47d42e820c7e7936f39e9d3c1d317
+  data.tar.gz: 7f742808cd90e3cdb2b983bced6a8a224a9be18c3a5121e679316edaba9648bff90711888d20fe06274c16eefe5340d6c0b52ef5c417fe1ebe0495167be23cae

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v0.11.1
+
+- Add `--repeat-result` option to return the best, the worst or an average result with `--repeat-count`
+- Add `BenchmarkDriver::BulkOutput` to make an output plugin casually
+
 # v0.11.0
 
 - [breaking change] Plugin interface is completely changed, so all plugins need migration

data/README.md

@@ -273,7 +273,7 @@ ips, time, memory, once
 | time | Elapsed seconds |
 | memory | Max resident set. This is supported only on Linux for now. |
 | once | Forces `loop_count` to 1 for testing |
-| command\_stdout | Special runner to integrate existing benchmarks |
+| ruby\_stdout | Special runner to integrate existing benchmarks |
 
 ### ips
 
@@ -353,12 +353,11 @@ Comparison:
                2.4.3:   1531393.6 i/s - 1.58x  slower
 ```
 
-### command\_stdout
+### ruby\_stdout
 
-See following examples:
+See following example:
 
 * https://github.com/benchmark-driver/optcarrot
-* https://github.com/benchmark-driver/fluentd-benchmark
 
 If you benchmark can run with `ruby foo bar`, specify `foo bar` to `command:`.
 Then write `stdout_to_metrics:` to convert stdout to metrics. This runner can be used only with YAML interface for now.

data/exe/benchmark-driver

@@ -42,6 +42,12 @@ config = BenchmarkDriver::Config.new.tap do |c|
         abort "-r, --repeat-count must take Integer, but got #{v.inspect}"
       end
     end
+    o.on('--repeat-result [TYPE]', 'Yield "best", "average" or "worst" result with --repeat-count (default: best)') do |v|
+      unless BenchmarkDriver::Repeater::VALID_TYPES.include?(v)
+        raise ArgumentError.new("--repeat-result must be #{BenchmarkDriver::Repeater::VALID_TYPES.join(', ')} but got #{v.inspect}")
+      end
+      c.repeat_result = v
+    end
     o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
       bundler = v
     end

data/lib/benchmark_driver.rb

@@ -1,7 +1,9 @@
+require 'benchmark_driver/bulk_output'
 require 'benchmark_driver/config'
 require 'benchmark_driver/job_parser'
 require 'benchmark_driver/output'
 require 'benchmark_driver/rbenv'
+require 'benchmark_driver/repeater'
 require 'benchmark_driver/ruby_interface'
 require 'benchmark_driver/runner'
 require 'benchmark_driver/version'

data/lib/benchmark_driver/bulk_output.rb

@@ -0,0 +1,62 @@
+module BenchmarkDriver
+  # This is API for your casual output plugin and NOT internally used by BenchmarkDriver.
+  #
+  # By fully utilizing with_*/report APIs, you can implement streaming-output plugins.
+  # See also: lib/benchmark_driver/output.rb (this class's instance will be `@output`)
+  # But using these APIs can be difficult because the API is not stable yet and it's hard
+  # to deal with the complex state machine.
+  #
+  # If you don't need to output results in a streaming manner, you can create an output
+  # plugin class that inherits `BenchmarkDriver::BulkOutput`, which requires to override
+  # only `#bulk_output` that takes all inputs at once.
+  class BulkOutput
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    attr_writer :metrics
+
+    # @param [Array<String>] job_names
+    # @param [Array<String>] context_names
+    def initialize(job_names:, context_names:)
+      # noop
+    end
+
+    # The main API you need to override if you make a class inherit `BenchmarkDriver::BulkOutput`.
+    # @param [Hash{ BenchmarkDriver::Job => Hash{ BenchmarkDriver::Context => { BenchmarkDriver::Metric => Float } } }] result
+    # @param [Array<BenchmarkDriver::Metric>] metrics
+    def bulk_output(result:, metrics:)
+      raise NotImplementedError.new("#{self.class} must override #bulk_output")
+    end
+
+    def with_warmup(&block)
+      block.call # noop
+    end
+
+    def with_benchmark(&block)
+      @result = Hash.new do |h1, job|
+        h1[job] = Hash.new do |h2, context|
+          h2[context] = {}
+        end
+      end
+      result = block.call
+      bulk_output(result: @result, metrics: @metrics)
+      result
+    end
+
+    # @param [BenchmarkDriver::Job] job
+    def with_job(job, &block)
+      @job = job
+      block.call
+    end
+
+    # @param [BenchmarkDriver::Context] context
+    def with_context(context, &block)
+      @context = context
+      block.call
+    end
+
+    # @param [Float] value
+    # @param [BenchmarkDriver::Metric] metic
+    def report(value:, metric:)
+      @result[@job][@context][metric] = value
+    end
+  end
+end

data/lib/benchmark_driver/config.rb

@@ -3,19 +3,21 @@ require 'benchmark_driver/struct'
 module BenchmarkDriver
   # All CLI options
   Config = ::BenchmarkDriver::Struct.new(
-    :runner_type,  # @param [String]
-    :output_type,  # @param [String]
-    :paths,        # @param [Array<String>]
-    :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
-    :filters,      # @param [Array<Regexp>]
-    :repeat_count, # @param [Integer]
-    :run_duration, # @param [Float]
-    :verbose,      # @param [Integer]
+    :runner_type,   # @param [String]
+    :output_type,   # @param [String]
+    :paths,         # @param [Array<String>]
+    :executables,   # @param [Array<BenchmarkDriver::Config::Executable>]
+    :filters,       # @param [Array<Regexp>]
+    :repeat_count,  # @param [Integer]
+    :repeat_result, # @param [String]
+    :run_duration,  # @param [Float]
+    :verbose,       # @param [Integer]
     defaults: {
       runner_type: 'ips',
       output_type: 'compare',
       filters: [],
       repeat_count: 1,
+      repeat_result: 'best',
       run_duration: 3.0,
       verbose: 0,
     },
@@ -23,10 +25,11 @@ module BenchmarkDriver
 
   # Subset of FullConfig passed to JobRunner
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
-    :executables,  # @param [Array<BenchmarkDriver::Config::Executable>]
-    :repeat_count, # @param [Integer]
-    :run_duration, # @param [Float]
-    :verbose,      # @param [Integer]
+    :executables,   # @param [Array<BenchmarkDriver::Config::Executable>]
+    :repeat_count,  # @param [Integer]
+    :repeat_result, # @param [String]
+    :run_duration,  # @param [Float]
+    :verbose,       # @param [Integer]
   )
 
   Config::Executable = ::BenchmarkDriver::Struct.new(

data/lib/benchmark_driver/repeater.rb

@@ -0,0 +1,52 @@
+module BenchmarkDriver
+  # Repeat calling block and return desired result: "best", "worst" or "average".
+  module Repeater
+    VALID_TYPES = %w[best worst average]
+
+    class << self
+      # `block.call` can return multiple objects, but the first one is used for sort.
+      # When `config.repeat_result == 'average'`, how to deal with rest objects is decided
+      # by `:rest_on_average` option.
+      def with_repeat(config:, larger_better:, rest_on_average: :first, &block)
+        values = config.repeat_count.times.map { block.call }
+
+        case config.repeat_result
+        when 'best'
+          best_result(values, larger_better)
+        when 'worst'
+          best_result(values, !larger_better)
+        when 'average'
+          average_result(values, rest_on_average)
+        else
+          raise "unexpected repeat_result #{config.repeat_result.inspect}"
+        end
+      end
+
+      private
+
+      def best_result(values, larger_better)
+        values.sort_by do |value, *|
+          larger_better ? value : -value
+        end.last
+      end
+
+      def average_result(values, rest_on_average)
+        unless values.first.is_a?(Array)
+          return values.inject(&:+) / values.size.to_f
+        end
+
+        case rest_on_average
+        when :first
+          rest = values.first[1..-1]
+          [values.map { |v| v[0] }.inject(&:+) / values.size.to_f, *rest]
+        when :average
+          values.first.size.times.map do |index|
+            values.map { |v| v[index] }.inject(&:+) / values.first.size.to_f
+          end
+        else
+          raise "unexpected rest_on_average #{rest_on_average.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/benchmark_driver/runner.rb

@@ -23,6 +23,7 @@ module BenchmarkDriver
       runner_config = Config::RunnerConfig.new(
         executables: config.executables,
         repeat_count: config.repeat_count,
+        repeat_result: config.repeat_result,
         run_duration: config.run_duration,
         verbose: config.verbose,
       )

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -62,7 +62,7 @@ class BenchmarkDriver::Runner::CommandStdout
       jobs.each do |job|
         @output.with_job(name: job.name) do
           @config.executables.each do |exec|
-            best_value = with_repeat(metric) do
+            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: metric.larger_better) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
               end
@@ -73,7 +73,7 @@ class BenchmarkDriver::Runner::CommandStdout
             end
 
             @output.with_context(name: exec.name, executable: exec) do
-              @output.report(value: best_value, metric: metric)
+              @output.report(value: value, metric: metric)
             end
           end
         end
@@ -107,20 +107,6 @@ class BenchmarkDriver::Runner::CommandStdout
     stdout
   end
 
-  # Return multiple times and return the best metrics
-  def with_repeat(metric, &block)
-    values = @config.repeat_count.times.map do
-      block.call
-    end
-    values.sort_by do |value|
-      if metric.larger_better
-        value
-      else
-        -value
-      end
-    end.last
-  end
-
   StdoutToMetrics = ::BenchmarkDriver::Struct.new(:stdout, :stdout_to_metrics) do
     def metrics_value
       eval(stdout_to_metrics, binding)

data/lib/benchmark_driver/runner/ips.rb

@@ -51,7 +51,8 @@ class BenchmarkDriver::Runner::Ips
       jobs.each do |job|
         @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
-            value, duration = with_repeat(@config.repeat_count) do
+            repeat_params = { config: @config, larger_better: true, rest_on_average: :average }
+            value, duration = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               run_benchmark(job, exec: exec)
             end
             @output.with_context(name: exec.name, executable: exec, duration: duration) do
@@ -87,16 +88,6 @@ class BenchmarkDriver::Runner::Ips
     [duration, loop_count]
   end
 
-  # Return multiple times and return the best metrics
-  def with_repeat(repeat_times, &block)
-    value_durations = repeat_times.times.map do
-      block.call
-    end
-    value_durations.sort_by do |value, _|
-      metric.larger_better ? value : -value
-    end.last
-  end
-
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
   # @param [BenchmarkDriver::Config::Executable] exec
   # @return [BenchmarkDriver::Metrics]

data/lib/benchmark_driver/runner/memory.rb

@@ -43,11 +43,11 @@ class BenchmarkDriver::Runner::Memory
       jobs.each do |job|
         @output.with_job(name: job.name) do
           job.runnable_execs(@config.executables).each do |exec|
-            best_value = with_repeat(@config.repeat_count) do
+            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
               run_benchmark(job, exec: exec)
             end
             @output.with_context(name: exec.name, executable: exec, loop_count: job.loop_count) do
-              @output.report(value: best_value, metric: METRIC)
+              @output.report(value: value, metric: METRIC)
             end
           end
         end
@@ -57,14 +57,6 @@ class BenchmarkDriver::Runner::Memory
 
   private
 
-  # Return multiple times and return the best value (smallest usage)
-  def with_repeat(repeat_times, &block)
-    values = repeat_times.times.map do
-      block.call
-    end
-    values.sort.first
-  end
-
   # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
   # @param [BenchmarkDriver::Config::Executable] exec
   # @return [BenchmarkDriver::Metrics]

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -79,7 +79,8 @@ class BenchmarkDriver::Runner::RubyStdout
       jobs.each do |job|
         @output.with_job(name: job.name) do
           @config.executables.each do |exec|
-            best_value, environment = with_repeat(metric) do
+            repeat_params = { config: @config, larger_better: metric.larger_better }
+            value, environment = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
               end
@@ -92,7 +93,7 @@ class BenchmarkDriver::Runner::RubyStdout
             end
 
             @output.with_context(name: exec.name, executable: exec, environment: environment) do
-              @output.report(value: best_value, metric: metric)
+              @output.report(value: value, metric: metric)
             end
           end
         end

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.11.0'
+  VERSION = '0.11.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.11.1
 platform: ruby
 authors:
 - Takashi Kokubun
@@ -99,6 +99,7 @@ files:
 - exe/benchmark-driver
 - images/optcarrot.png
 - lib/benchmark_driver.rb
+- lib/benchmark_driver/bulk_output.rb
 - lib/benchmark_driver/config.rb
 - lib/benchmark_driver/default_job.rb
 - lib/benchmark_driver/default_job_parser.rb
@@ -110,6 +111,7 @@ files:
 - lib/benchmark_driver/output/record.rb
 - lib/benchmark_driver/output/simple.rb
 - lib/benchmark_driver/rbenv.rb
+- lib/benchmark_driver/repeater.rb
 - lib/benchmark_driver/ruby_interface.rb
 - lib/benchmark_driver/runner.rb
 - lib/benchmark_driver/runner/command_stdout.rb