benchmark_driver 0.15.15 → 0.15.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6a5ecae78649aa29ad4d54635e170cb6ab998cc1f4e67102a821dd39a5646ac
-  data.tar.gz: 0c131020c654dc8b4fb6cf40dd134fc959267602722d8eaa538f39c2e99768cd
+  metadata.gz: 264d1545a3f32fef38774946fc4c5dcb5678605687709b18d122baed38b0a6c3
+  data.tar.gz: 96eae9e96e0aaa018e8142a6299c644b14306430c4b49885a5e324c2679d33ac
 SHA512:
-  metadata.gz: 1e471938da9128c589182326c6dcbc6a0eb62249150165452b96635f598c2ad8b3e322107ee41b51f440d6ddaaffd5735d5b8046a660abdfaea9cdf64322d433
-  data.tar.gz: 3f4ce47df9e8d5b62c3bd73aef5ebe89976b44886ccf94bd3d78b9f58a62538744f62b6dd709eb70216257183efa888d0e1be1ed5cac29e4a77c2a8498002dde
+  metadata.gz: 2e466c27cb262ca00ce2ed36806b26c28d1313432796a6f561d3374223d9ea9fa844a264dd3888637f76c9b871adaf1ace2fae7871926287643d4ecabcf7bbfb
+  data.tar.gz: 6f8795a33250a1ab8ce2a909068a50d35280cb57b0878958a8f059019a464b74c209d1c082d327c908d8755b5ddf771c13c5d785edcd46a0d7db038774ddb9f3

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v0.15.16
+
+- Add `--alternate` runner option to run executables alternately.
+  This is supported only for `ruby_stdout` runner for now.
+
 # v0.15.15
 
 - An absolute path is passed to `command_stdout`'s `working_directory`

data/exe/benchmark-driver

@@ -67,6 +67,9 @@ config = BenchmarkDriver::Config.new.tap do |c|
       end
       c.repeat_result = v
     end
+    o.on('--alternate', 'Alternate executables instead of running the same executable in a row with --repeat-count') do |v|
+      c.alternate = v
+    end
     o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
       bundler = v
     end

data/lib/benchmark_driver/config.rb

@@ -13,6 +13,7 @@ module BenchmarkDriver
     :filters,       # @param [Array<Regexp>]
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]
@@ -23,15 +24,17 @@ module BenchmarkDriver
       filters: [],
       repeat_count: 1,
       repeat_result: 'best',
+      alternate: false,
       run_duration: 3.0,
       verbose: 0,
     },
   )
 
-  # Subset of FullConfig passed to JobRunner
+  # Subset of Config passed to JobRunner
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]

data/lib/benchmark_driver/output/markdown.rb

@@ -8,6 +8,7 @@ class BenchmarkDriver::Output::Markdown
   # @param [Array<BenchmarkDriver::Metric>] metrics
   # @param [Array<BenchmarkDriver::Job>] jobs
   # @param [Array<BenchmarkDriver::Context>] contexts
+  # @param [Hash{ Symbol => Object }] options
   def initialize(metrics:, jobs:, contexts:, options:)
     @metrics = metrics
     @contexts = contexts

data/lib/benchmark_driver/runner.rb

@@ -30,6 +30,10 @@ module BenchmarkDriver
         contexts_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
           metrics_jobs.group_by(&:class).each do |klass, klass_jobs|
             runner = runner_for(klass)
+            if runner_config.alternate && runner != BenchmarkDriver::Runner::RubyStdout
+              abort "--alternate is supported only for ruby_stdout runner for now"
+            end
+
             contexts = build_contexts(contexts, executables: config.executables)
             output = Output.new(
               type: config.output_type,

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -77,6 +77,63 @@ class BenchmarkDriver::Runner::RubyStdout
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
+    if @config.alternate
+      alternated_run(jobs)
+    else
+      incremental_run(jobs)
+    end
+  end
+
+  private
+
+  # Special mode. Execution order: RubyA, RubyB, ..., RubyA, RubyB, ...
+  def alternated_run(jobs)
+    metric = jobs.first.metrics.first
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          # Running benchmarks in an alternated manner is NOT compatible with two things:
+          #   * Output plugins. They expect RubyA, RubyA, RubyB, RubyB, ...
+          #   * BenchmarkDriver::Repeater. It should be used for results of the same condition.
+          #
+          # Therefore, we run all benchmarks with executables alternated first here, and then
+          # aggregate the results as if the same executable were repeated in a row.
+          context_results = Hash.new do |hash, context|
+            hash[context] = []
+          end
+          jobs.each do |job|
+            @config.repeat_count.times do
+              @contexts.each do |context|
+                context_results[context] << run_job(job, exec: context.executable)
+              end
+            end
+          end
+
+          # Aggregate reslts by BenchmarkDriver::Repeater and pass them to output.
+          @contexts.each do |context|
+            repeat_params = { config: @config, larger_better: metric.larger_better }
+            result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
+              context_results[context].shift
+            end
+            value, environment = result.value
+
+            exec = context.executable
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(
+                values: { metric => value },
+                all_values: { metric => result.all_values },
+                environment: environment,
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+
+  # Default mode. Execution order: RubyA, RubyA, RubyB, RubyB, ...
+  def incremental_run(jobs)
     metric = jobs.first.metrics.first
 
     @output.with_benchmark do
@@ -86,20 +143,7 @@ class BenchmarkDriver::Runner::RubyStdout
             exec = context.executable
             repeat_params = { config: @config, larger_better: metric.larger_better }
             result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
-              begin
-                stdout = with_chdir(job.working_directory) do
-                  with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
-                end
-                script = StdoutToMetrics.new(
-                  stdout: stdout,
-                  value_from_stdout: job.value_from_stdout,
-                  environment_from_stdout: job.environment_from_stdout,
-                )
-                [script.value, script.environment]
-              rescue CommandFailure => e
-                $stderr.puts("\n```\n#{e.message}```\n")
-                [BenchmarkDriver::Result::ERROR, {}]
-              end
+              run_job(job, exec: exec)
             end
             value, environment = result.value
 
@@ -116,7 +160,21 @@ class BenchmarkDriver::Runner::RubyStdout
     end
   end
 
-  private
+  # Run a job and return what BenchmarkDriver::Repeater.with_repeat takes.
+  def run_job(job, exec:)
+    stdout = with_chdir(job.working_directory) do
+      with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
+    end
+    script = StdoutToMetrics.new(
+      stdout: stdout,
+      value_from_stdout: job.value_from_stdout,
+      environment_from_stdout: job.environment_from_stdout,
+    )
+    [script.value, script.environment]
+  rescue CommandFailure => e
+    $stderr.puts("\n```\n#{e.message}```\n")
+    [BenchmarkDriver::Result::ERROR, {}]
+  end
 
   def with_ruby_prefix(executable, &block)
     env = ENV.to_h.dup

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.15.15'
+  VERSION = '0.15.16'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.15.15
+  version: 0.15.16
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-22 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0.pre1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby