benchmark_driver 0.15.12 → 0.15.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c069979523cfd025fba37653a117db89a52062bfedd7fe3d2c8b8ef5eaab329
-  data.tar.gz: 23f32d6a140120ccbddd2ce704590cde72496fc0e7834674fecc865d04e8c943
+  metadata.gz: cc6888fbbe3ec73fcb84874bbc31535a08ae07859cbc8b86c0dd53d2e33ed2b7
+  data.tar.gz: cc052011eb026f8dd4d250a191fcbe30bdb3024b2157659c80696649d0c58840
 SHA512:
-  metadata.gz: d9d9a2cde658660d21dcf36a2ff872ac42c7a518aaf2a917744eacbcf8e2e95814ded6edb140a3b9d23273291850a2948cc47c09297d5b973b531da3a2582fd5
-  data.tar.gz: 48a7c466c98a4304c3cc44bcfc7adf2d277c546e226cb3082999a1a384f038f969c7782b338b851bafd9c319d2d1aad4adca243b1c358346a2b79c1ca0ac0c78
+  metadata.gz: 2516d64721d7b53f1466468263464024ebcdcc9906787ff29b38a494e2f8ecab275ccad7fcdc4928c1482c43b4ec0195d2733b38476ca919ff95b3b304cc4174
+  data.tar.gz: e6e62c506e94fe8e2add9af0ea3bd6abd531299117d28e7a2b1cbfee1d56db309907d1787b6cb090bdd64cefe0206641ec753b15ceb0625ed118406e61af1d33

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+# v0.15.17
+
+- Jobs returned from job parsers are made mutable for plugins
+
+# v0.15.16
+
+- Add `--alternate` runner option to run executables alternately.
+  This is supported only for `ruby_stdout` runner for now.
+
+# v0.15.15
+
+- An absolute path is passed to `command_stdout`'s `working_directory`
+  and v0.15.14's relative path resolution of job type
+
+# v0.15.14
+
+- YAML's `type` key allows a value with `/` to specify a relative path of a runner plugin
+
+# v0.15.13
+
+- Show a command and stdout on `-vv` for `ips`, `time`, and `block` runner
+
 # v0.15.12
 
 - Show comparison on `--output=markdown` when `--output-compare` is also specified

data/exe/benchmark-driver

@@ -67,6 +67,9 @@ config = BenchmarkDriver::Config.new.tap do |c|
       end
       c.repeat_result = v
     end
+    o.on('--alternate', 'Alternate executables instead of running the same executable in a row with --repeat-count') do |v|
+      c.alternate = v
+    end
     o.on('--bundler', 'Install and use gems specified in Gemfile') do |v|
       bundler = v
     end
@@ -124,7 +127,7 @@ jobs = config.paths.flat_map do |path|
 
   begin
     # `working_directory` is YAML-specific special parameter, mainly for "command_stdout"
-    BenchmarkDriver::JobParser.parse(job, default_params: { working_directory: File.dirname(path) })
+    BenchmarkDriver::JobParser.parse(job, working_directory: File.expand_path(File.dirname(path)))
   rescue ArgumentError
     $stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
     $stderr.puts '  YAML format may be wrong. See error below:'

data/lib/benchmark_driver/config.rb

@@ -13,6 +13,7 @@ module BenchmarkDriver
     :filters,       # @param [Array<Regexp>]
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]
@@ -23,15 +24,17 @@ module BenchmarkDriver
       filters: [],
       repeat_count: 1,
       repeat_result: 'best',
+      alternate: false,
       run_duration: 3.0,
       verbose: 0,
     },
   )
 
-  # Subset of FullConfig passed to JobRunner
+  # Subset of Config passed to JobRunner
   Config::RunnerConfig = ::BenchmarkDriver::Struct.new(
     :repeat_count,  # @param [Integer]
     :repeat_result, # @param [String]
+    :alternate,     # @param [TrueClass,FalseClass]
     :run_duration,  # @param [Float]
     :timeout,       # @param [Float,nil]
     :verbose,       # @param [Integer]

data/lib/benchmark_driver/default_job_parser.rb

@@ -26,7 +26,7 @@ module BenchmarkDriver
         job.teardown.prepend("#{teardown}\n") if teardown
         job.loop_count ||= loop_count
         job.required_ruby_version ||= required_ruby_version
-      end.each(&:freeze)
+      end
     end
 
     private

data/lib/benchmark_driver/job_parser.rb

@@ -3,24 +3,27 @@ require 'benchmark_driver/runner'
 module BenchmarkDriver
   class << JobParser = Module.new
     # @param [Hash] config
-    # @param [Hash] default_params - Special default values not written in job definition
-    def parse(config, default_params: {})
+    # @param [Hash] working_directory - YAML-specific special parameter for "command_stdout" and a relative path in type
+    def parse(config, working_directory: nil)
       config = symbolize_keys(config)
       type = config.fetch(:type)
       if !type.is_a?(String)
         raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
-      elsif !type.match(/\A[A-Za-z0-9_]+\z/)
-        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
+      elsif !type.match(/\A[A-Za-z0-9_\/]+\z/)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_\/])")
       end
       config.delete(:type)
 
       # Dynamic dispatch for plugin support
-      require "benchmark_driver/runner/#{type}"
+      if type.include?('/')
+        require File.join(working_directory || '.', type)
+        type = File.basename(type)
+      else
+        require "benchmark_driver/runner/#{type}"
+      end
       job = ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(**config)
-      default_params.each do |key, value|
-        if job.respond_to?(key) && job.respond_to?("#{key}=") && job.public_send(key).nil?
-          job.public_send("#{key}=", value)
-        end
+      if job.respond_to?(:working_directory) && job.respond_to?(:working_directory=) && job.working_directory.nil?
+        job.working_directory = working_directory
       end
       job
     end

data/lib/benchmark_driver/output/markdown.rb

@@ -8,6 +8,7 @@ class BenchmarkDriver::Output::Markdown
   # @param [Array<BenchmarkDriver::Metric>] metrics
   # @param [Array<BenchmarkDriver::Job>] jobs
   # @param [Array<BenchmarkDriver::Context>] contexts
+  # @param [Hash{ Symbol => Object }] options
   def initialize(metrics:, jobs:, contexts:, options:)
     @metrics = metrics
     @contexts = contexts

data/lib/benchmark_driver/runner.rb

@@ -30,6 +30,10 @@ module BenchmarkDriver
         contexts_jobs.group_by(&:metrics).each do |metrics, metrics_jobs|
           metrics_jobs.group_by(&:class).each do |klass, klass_jobs|
             runner = runner_for(klass)
+            if runner_config.alternate && runner != BenchmarkDriver::Runner::RubyStdout
+              abort "--alternate is supported only for ruby_stdout runner for now"
+            end
+
             contexts = build_contexts(contexts, executables: config.executables)
             output = Output.new(
               type: config.output_type,

data/lib/benchmark_driver/runner/ips.rb

@@ -40,8 +40,8 @@ class BenchmarkDriver::Runner::Ips
               @output.report(values: { metric => value }, duration: duration, loop_count: loop_count)
             end
 
-            loop_count = (loop_count.to_f * @config.run_duration / duration).floor
-            Job.new(**job.to_h.merge(loop_count: loop_count))
+            job.loop_count = (loop_count.to_f * @config.run_duration / duration).floor
+            job
           end
         end
       end
@@ -107,8 +107,8 @@ class BenchmarkDriver::Runner::Ips
 
     duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
-        if $?.success? && ((value = Float(f.read)) > 0)
+        success = execute(*context.executable.command, path, exception: false)
+        if success && ((value = Float(f.read)) > 0)
           value
         else
           BenchmarkDriver::Result::ERROR
@@ -137,10 +137,7 @@ class BenchmarkDriver::Runner::Ips
   end
 
   def with_script(script)
-    if @config.verbose >= 2
-      sep = '-' * 30
-      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
-    end
+    debug_output('Script', script) if @config.verbose >= 2
 
     Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       f.puts script
@@ -149,11 +146,20 @@ class BenchmarkDriver::Runner::Ips
     end
   end
 
-  def execute(*args)
-    IO.popen(args, &:read) # TODO: print stdout if verbose=2
-    unless $?.success?
-      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+  def execute(*args, exception: true)
+    $stderr.puts "$ #{args.shelljoin}" if @config.verbose >= 2
+
+    stdout = IO.popen(args, &:read)
+    debug_output('Command output', stdout) if @config.verbose >= 2
+    if exception && !$?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?})"
     end
+    $?.success?
+  end
+
+  def debug_output(name, text)
+    sep = '-' * 30
+    $stdout.puts "\n\n#{sep}[#{name} begin]#{sep}\n#{text}#{sep}[#{name} end]#{sep}\n\n"
   end
 
   WarmupScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count, :first_warmup_duration, :second_warmup_duration) do

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -77,6 +77,63 @@ class BenchmarkDriver::Runner::RubyStdout
   # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
   # @param [Array<BenchmarkDriver::Default::Job>] jobs
   def run(jobs)
+    if @config.alternate
+      alternated_run(jobs)
+    else
+      incremental_run(jobs)
+    end
+  end
+
+  private
+
+  # Special mode. Execution order: RubyA, RubyB, ..., RubyA, RubyB, ...
+  def alternated_run(jobs)
+    metric = jobs.first.metrics.first
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          # Running benchmarks in an alternated manner is NOT compatible with two things:
+          #   * Output plugins. They expect RubyA, RubyA, RubyB, RubyB, ...
+          #   * BenchmarkDriver::Repeater. It should be used for results of the same condition.
+          #
+          # Therefore, we run all benchmarks with executables alternated first here, and then
+          # aggregate the results as if the same executable were repeated in a row.
+          context_results = Hash.new do |hash, context|
+            hash[context] = []
+          end
+          jobs.each do |job|
+            @config.repeat_count.times do
+              @contexts.each do |context|
+                context_results[context] << run_job(job, exec: context.executable)
+              end
+            end
+          end
+
+          # Aggregate reslts by BenchmarkDriver::Repeater and pass them to output.
+          @contexts.each do |context|
+            repeat_params = { config: @config, larger_better: metric.larger_better }
+            result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
+              context_results[context].shift
+            end
+            value, environment = result.value
+
+            exec = context.executable
+            @output.with_context(name: exec.name, executable: exec) do
+              @output.report(
+                values: { metric => value },
+                all_values: { metric => result.all_values },
+                environment: environment,
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+
+  # Default mode. Execution order: RubyA, RubyA, RubyB, RubyB, ...
+  def incremental_run(jobs)
     metric = jobs.first.metrics.first
 
     @output.with_benchmark do
@@ -86,20 +143,7 @@ class BenchmarkDriver::Runner::RubyStdout
             exec = context.executable
             repeat_params = { config: @config, larger_better: metric.larger_better }
             result = BenchmarkDriver::Repeater.with_repeat(**repeat_params) do
-              begin
-                stdout = with_chdir(job.working_directory) do
-                  with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
-                end
-                script = StdoutToMetrics.new(
-                  stdout: stdout,
-                  value_from_stdout: job.value_from_stdout,
-                  environment_from_stdout: job.environment_from_stdout,
-                )
-                [script.value, script.environment]
-              rescue CommandFailure => e
-                $stderr.puts("\n```\n#{e.message}```\n")
-                [BenchmarkDriver::Result::ERROR, {}]
-              end
+              run_job(job, exec: exec)
             end
             value, environment = result.value
 
@@ -116,7 +160,21 @@ class BenchmarkDriver::Runner::RubyStdout
     end
   end
 
-  private
+  # Run a job and return what BenchmarkDriver::Repeater.with_repeat takes.
+  def run_job(job, exec:)
+    stdout = with_chdir(job.working_directory) do
+      with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
+    end
+    script = StdoutToMetrics.new(
+      stdout: stdout,
+      value_from_stdout: job.value_from_stdout,
+      environment_from_stdout: job.environment_from_stdout,
+    )
+    [script.value, script.environment]
+  rescue CommandFailure => e
+    $stderr.puts("\n```\n#{e.message}```\n")
+    [BenchmarkDriver::Result::ERROR, {}]
+  end
 
   def with_ruby_prefix(executable, &block)
     env = ENV.to_h.dup

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.15.12'
+  VERSION = '0.15.17'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.15.12
+  version: 0.15.17
 platform: ruby
 authors:
 - Takashi Kokubun
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2021-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -131,7 +131,7 @@ homepage: https://github.com/benchmark-driver/benchmark-driver
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -146,8 +146,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.0.pre1
-signing_key: 
+rubygems_version: 3.2.6
+signing_key:
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby
 test_files: []