benchmark_driver 0.15.15 → 0.15.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6a5ecae78649aa29ad4d54635e170cb6ab998cc1f4e67102a821dd39a5646ac
-  data.tar.gz: 0c131020c654dc8b4fb6cf40dd134fc959267602722d8eaa538f39c2e99768cd
+  metadata.gz: 056db37cfc1c2453c9ab994d4b1241bb450ad7e138e69e2ae4a5995825976753
+  data.tar.gz: f827c814001e175ebcb878181ff0b6b411466bb34bf4ff4a34e02b0f1fdafdda
 SHA512:
-  metadata.gz: 1e471938da9128c589182326c6dcbc6a0eb62249150165452b96635f598c2ad8b3e322107ee41b51f440d6ddaaffd5735d5b8046a660abdfaea9cdf64322d433
-  data.tar.gz: 3f4ce47df9e8d5b62c3bd73aef5ebe89976b44886ccf94bd3d78b9f58a62538744f62b6dd709eb70216257183efa888d0e1be1ed5cac29e4a77c2a8498002dde
+  metadata.gz: f34f11c0d5a30be2e232af08d00e2e1b2420a161aa18c1f020a0f342f53804b82da754c61632a008ec7822f710edab2af7a1dde18cf1a67a4e51ab14852b3b3a
+  data.tar.gz: e0601d752e1f55d95ae2c1b83f6dfaa432290b28856ceca553500597f8b06b508c34909df98cd96f3aaf0989df9d2ef76fa0c1e31099f80486b37fa641ce15dd

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+# v0.15.18
+
+- Mention `--output=all` in help
+
+# v0.15.17
+
+- Jobs returned from job parsers are made mutable for plugins
+
+# v0.15.16
+
+- Add `--alternate` runner option to run executables alternately.
+  This is supported only for `ruby_stdout` runner for now.
+
 # v0.15.15
 
 - An absolute path is passed to `command_stdout`'s `working_directory`

data/exe/benchmark-driver

@@ -18,7 +18,7 @@ config = BenchmarkDriver::Config.new.tap do |c|
     o.on('-r', '--runner TYPE', String, 'Specify runner type: ips, time, memory, once, block (default: ips)') do |d|
       c.runner_type = d
     end
-    o.on('-o', '--output TYPE', String, 'Specify output type: compare, simple, markdown, record (default: compare)') do |out|
+    o.on('-o', '--output TYPE', String, 'Specify output type: compare, simple, markdown, record, all (default: compare)') do |out|
       c.output_type = out
       begin
         plugin_options = BenchmarkDriver::Output.get(out).const_get('OPTIONS', false)
@@ -67,6 +67,9 @@ config = BenchmarkDriver::Config.new.tap do |c|
       end
       c.repeat_result = v
     end
+    o.on('--alternate', 'Alternate executables instead of running the same executable in a row with --repeat-count') do |v|
+      c.alternate = v
+    end
     o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
       bundler = v
     end

data/lib/benchmark_driver/config.rb

@@ -13,6 +13,7 @@ module BenchmarkDriver
     :filters,       # @param [Array<Regexp>]
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]
@@ -23,15 +24,17 @@ module BenchmarkDriver
       filters: [],
       repeat_count: 1,
       repeat_result: 'best',
+      alternate: false,
       run_duration: 3.0,
       verbose: 0,
     },
   )
 
-  # Subset of FullConfig passed to JobRunner
+  # Subset of Config passed to JobRunner
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]

data/lib/benchmark_driver/default_job_parser.rb

@@ -26,7 +26,7 @@ module BenchmarkDriver
         job.teardown.prepend("#{teardown}\n") if teardown
         job.loop_count ||= loop_count
         job.required_ruby_version ||= required_ruby_version
-      end.each(&:freeze)
+      end
     end
 
     private

data/lib/benchmark_driver/output/markdown.rb

@@ -8,6 +8,7 @@ class BenchmarkDriver::Output::Markdown
   # @param [Array<BenchmarkDriver::Metric>] metrics
   # @param [Array<BenchmarkDriver::Job>] jobs
   # @param [Array<BenchmarkDriver::Context>] contexts
+  # @param [Hash{ Symbol => Object }] options
   def initialize(metrics:, jobs:, contexts:, options:)
     @metrics = metrics
     @contexts = contexts

data/lib/benchmark_driver/runner/ips.rb

@@ -40,8 +40,8 @@ class BenchmarkDriver::Runner::Ips
               @output.report(values: { metric => value }, duration: duration, loop_count: loop_count)
             end
 
-            loop_count = (loop_count.to_f * @config.run_duration / duration).floor
-            Job.new(**job.to_h.merge(loop_count: loop_count))
+            job.loop_count = (loop_count.to_f * @config.run_duration / duration).floor
+            job
           end
         end
       end

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -77,6 +77,63 @@ class BenchmarkDriver::Runner::RubyStdout
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
+    if @config.alternate
+      alternated_run(jobs)
+    else
+      incremental_run(jobs)
+    end
+  end
+
+  private
+
+  # Special mode. Execution order: RubyA, RubyB, ..., RubyA, RubyB, ...
+  def alternated_run(jobs)
+    metric = jobs.first.metrics.first
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          # Running benchmarks in an alternated manner is NOT compatible with two things:
+          #   * Output plugins. They expect RubyA, RubyA, RubyB, RubyB, ...
+          #   * BenchmarkDriver::Repeater. It should be used for results of the same condition.
+          #
+          # Therefore, we run all benchmarks with executables alternated first here, and then
+          # aggregate the results as if the same executable were repeated in a row.
+          context_results = Hash.new do |hash, context|
+            hash[context] = []
+          end
+          jobs.each do |job|
+            @config.repeat_count.times do
+              @contexts.each do |context|
+                context_results[context] << run_job(job, exec: context.executable)
+              end
+            end
+          end
+
+          # Aggregate reslts by BenchmarkDriver::Repeater and pass them to output.
+          @contexts.each do |context|
+            repeat_params = { config: @config, larger_better: metric.larger_better }
+            result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
+              context_results[context].shift
+            end
+            value, environment = result.value
+
+            exec = context.executable
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(
+                values: { metric => value },
+                all_values: { metric => result.all_values },
+                environment: environment,
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+
+  # Default mode. Execution order: RubyA, RubyA, RubyB, RubyB, ...
+  def incremental_run(jobs)
     metric = jobs.first.metrics.first
 
     @output.with_benchmark do
@@ -86,20 +143,7 @@ class BenchmarkDriver::Runner::RubyStdout
             exec = context.executable
             repeat_params = { config: @config, larger_better: metric.larger_better }
             result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
-              begin
-                stdout = with_chdir(job.working_directory) do
-                  with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
-                end
-                script = StdoutToMetrics.new(
-                  stdout: stdout,
-                  value_from_stdout: job.value_from_stdout,
-                  environment_from_stdout: job.environment_from_stdout,
-                )
-                [script.value, script.environment]
-              rescue CommandFailure => e
-                $stderr.puts("\n```\n#{e.message}```\n")
-                [BenchmarkDriver::Result::ERROR, {}]
-              end
+              run_job(job, exec: exec)
             end
             value, environment = result.value
 
@@ -116,7 +160,21 @@ class BenchmarkDriver::Runner::RubyStdout
     end
   end
 
-  private
+  # Run a job and return what BenchmarkDriver::Repeater.with_repeat takes.
+  def run_job(job, exec:)
+    stdout = with_chdir(job.working_directory) do
+      with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
+    end
+    script = StdoutToMetrics.new(
+      stdout: stdout,
+      value_from_stdout: job.value_from_stdout,
+      environment_from_stdout: job.environment_from_stdout,
+    )
+    [script.value, script.environment]
+  rescue CommandFailure => e
+    $stderr.puts("\n```\n#{e.message}```\n")
+    [BenchmarkDriver::Result::ERROR, {}]
+  end
 
   def with_ruby_prefix(executable, &block)
     env = ENV.to_h.dup

data/lib/benchmark_driver/runner.rb

@@ -30,6 +30,10 @@ module BenchmarkDriver
         contexts_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
           metrics_jobs.group_by(&:class).each do |klass, klass_jobs|
             runner = runner_for(klass)
+            if runner_config.alternate && runner != BenchmarkDriver::Runner::RubyStdout
+              abort "--alternate is supported only for ruby_stdout runner for now"
+            end
+
             contexts = build_contexts(contexts, executables: config.executables)
             output = Output.new(
               type: config.output_type,

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.15.15'
+  VERSION = '0.15.18'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.15.15
+  version: 0.15.18
 platform: ruby
 authors:
 - Takashi Kokubun
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-22 00:00:00.000000000 Z
+date: 2022-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -131,7 +131,7 @@ homepage: https://github.com/benchmark-driver/benchmark-driver
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -146,8 +146,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0.pre1
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby
 test_files: []