benchmark_driver 0.15.10 → 0.15.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaaa85272039c5f803d0697fa8e5bc3dc9d8a0afbb3f638aca696862682117f7
-  data.tar.gz: 400d6628114c4c7d705e3e686ef296ddb31b065f914bb05b21ab4b9155c70783
+  metadata.gz: d6a5ecae78649aa29ad4d54635e170cb6ab998cc1f4e67102a821dd39a5646ac
+  data.tar.gz: 0c131020c654dc8b4fb6cf40dd134fc959267602722d8eaa538f39c2e99768cd
 SHA512:
-  metadata.gz: a781dcd1f68f4ca114c003378260534d355d8f907c1d0cc9208a73ec2179af21ace25a472f7d5322c06f68a7c1fb453602f09e91c52bf3da25b85d98ec83aea8
-  data.tar.gz: 7ef03da571389fd7e79cb7301adf042fbb10a10ffc6ffb22aa0f0e9d9e4d302029e5b30cbf4dbb097f2c5d9c9d654f81031f83bdeed8cccf7298bf3539e5ace6
+  metadata.gz: 1e471938da9128c589182326c6dcbc6a0eb62249150165452b96635f598c2ad8b3e322107ee41b51f440d6ddaaffd5735d5b8046a660abdfaea9cdf64322d433
+  data.tar.gz: 3f4ce47df9e8d5b62c3bd73aef5ebe89976b44886ccf94bd3d78b9f58a62538744f62b6dd709eb70216257183efa888d0e1be1ed5cac29e4a77c2a8498002dde

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+# v0.15.15
+
+- An absolute path is passed to `command_stdout`'s `working_directory`
+  and v0.15.14's relative path resolution of job type
+
+# v0.15.14
+
+- YAML's `type` key allows a value with `/` to specify a relative path of a runner plugin
+
+# v0.15.13
+
+- Show a command and stdout on `-vv` for `ips`, `time`, and `block` runner
+
+# v0.15.12
+
+- Show comparison on `--output=markdown` when `--output-compare` is also specified
+
+# v0.15.11
+
+- Correctly calculate an average result on `--repeat-result=average`
+
 # v0.15.10
 
 - Right-justify calculated results in `markdown` output

data/exe/benchmark-driver

@@ -124,7 +124,7 @@ jobs = config.paths.flat_map do |path|
 
   begin
     # `working_directory` is YAML-specific special parameter, mainly for "command_stdout"
-    BenchmarkDriver::JobParser.parse(job, default_params: { working_directory: File.dirname(path) })
+    BenchmarkDriver::JobParser.parse(job, working_directory: File.expand_path(File.dirname(path)))
   rescue ArgumentError
     $stderr.puts "benchmark-driver: Failed to parse #{path.dump}."
     $stderr.puts '  YAML format may be wrong. See error below:'

data/lib/benchmark_driver/job_parser.rb

@@ -3,24 +3,27 @@ require 'benchmark_driver/runner'
 module BenchmarkDriver
   class << JobParser = Module.new
     # @param [Hash] config
-    # @param [Hash] default_params - Special default values not written in job definition
-    def parse(config, default_params: {})
+    # @param [Hash] working_directory - YAML-specific special parameter for "command_stdout" and a relative path in type
+    def parse(config, working_directory: nil)
       config = symbolize_keys(config)
       type = config.fetch(:type)
       if !type.is_a?(String)
         raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected String)")
-      elsif !type.match(/\A[A-Za-z0-9_]+\z/)
-        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_])")
+      elsif !type.match(/\A[A-Za-z0-9_\/]+\z/)
+        raise ArgumentError.new("Invalid type: #{config[:type].inspect} (expected to include only [A-Za-z0-9_\/])")
       end
       config.delete(:type)
 
       # Dynamic dispatch for plugin support
-      require "benchmark_driver/runner/#{type}"
+      if type.include?('/')
+        require File.join(working_directory || '.', type)
+        type = File.basename(type)
+      else
+        require "benchmark_driver/runner/#{type}"
+      end
       job = ::BenchmarkDriver.const_get("Runner::#{camelize(type)}::JobParser", false).parse(**config)
-      default_params.each do |key, value|
-        if job.respond_to?(key) && job.respond_to?("#{key}=") && job.public_send(key).nil?
-          job.public_send("#{key}=", value)
-        end
+      if job.respond_to?(:working_directory) && job.respond_to?(:working_directory=) && job.working_directory.nil?
+        job.working_directory = working_directory
       end
       job
     end

data/lib/benchmark_driver/output/compare.rb

@@ -117,17 +117,17 @@ class BenchmarkDriver::Output::Compare
 
   def humanize(value, width = 10)
     if BenchmarkDriver::Result::ERROR.equal?(value)
-      return " %#{width}s" % 'ERROR'
+      return sprintf(" %*s", width, 'ERROR')
     elsif value == 0.0
-      return " %#{width}.3f" % 0.0
+      return sprintf(" %*.3f", width, 0.0)
     elsif value < 0
       raise ArgumentError.new("Negative value: #{value.inspect}")
     end
 
     scale = (Math.log10(value) / 3).to_i
-    return "%#{width}s" % value.to_s if scale < 0 # like 1.23e-04
+    return sprintf("%*s", width, value.to_s) if scale < 0 # like 1.23e-04
 
-    prefix = "%#{width}.3f" % (value.to_f / (1000 ** scale))
+    prefix = sprintf("%*.3f", width, (value.to_f / (1000 ** scale)))
     suffix =
       case scale
       when 1; 'k'
@@ -173,7 +173,7 @@ class BenchmarkDriver::Output::Compare
     $stdout.puts "\nComparison:"
 
     @job_context_result.each do |job, context_result|
-      $stdout.puts("%#{@name_length + 2 + 11}s" % job)
+      $stdout.printf("%*s\n", @name_length + 2 + 11, job)
       results = context_result.flat_map do |context, result|
         result.values.values.map { |value| Result.new(job: job, value: value, context: context) }
       end
@@ -188,7 +188,7 @@ class BenchmarkDriver::Output::Compare
 
     unless BenchmarkDriver::Result::ERROR.equal?(bottom)
       ratio = top / bottom
-      "- %.2fx  #{@metrics.first.worse_word}" % ratio
+      sprintf("- %.2fx  %s", ratio, @metrics.first.worse_word)
     end
   end
 
@@ -211,7 +211,7 @@ class BenchmarkDriver::Output::Compare
       else
         name = result.job
       end
-      $stdout.puts("%#{@name_length}s: %11.1f %s #{slower}" % [name, result.value, @metrics.first.unit])
+      $stdout.printf("%*s: %11.1f %s %s\n", @name_length, name, result.value, @metrics.first.unit, slower)
     end
     $stdout.puts
   end

data/lib/benchmark_driver/output/markdown.rb

@@ -1,13 +1,19 @@
 class BenchmarkDriver::Output::Markdown
   NAME_LENGTH = 8
 
+  OPTIONS = {
+    compare: ['--output-compare', 'Show comparison between results'],
+  }
+
   # @param [Array<BenchmarkDriver::Metric>] metrics
   # @param [Array<BenchmarkDriver::Job>] jobs
   # @param [Array<BenchmarkDriver::Context>] contexts
-  def initialize(metrics:, jobs:, contexts:)
+  def initialize(metrics:, jobs:, contexts:, options:)
     @metrics = metrics
+    @contexts = contexts
     @context_names = contexts.map(&:name)
     @name_length = jobs.map(&:name).map(&:size).max
+    @compare = options.fetch(:compare, false)
   end
 
   def with_warmup(&block)
@@ -28,14 +34,15 @@ class BenchmarkDriver::Output::Markdown
       # Show executable names
       $stdout.print("|#{' ' * @name_length}  ")
       @context_names.each do |context_name|
-        $stdout.print("|%#{NAME_LENGTH}s" % context_name) # same size as humanize
+        $stdout.printf("|%*s", NAME_LENGTH, context_name) # same size as humanize
       end
       $stdout.puts('|')
 
       # Show header separator
       $stdout.print("|:#{'-' * (@name_length - 1)}--")
       @context_names.each do |context_name|
-        $stdout.print("|#{'-' * (NAME_LENGTH - 1)}:") # same size as humanize
+        length = [context_name.length, NAME_LENGTH].max
+        $stdout.print("|#{'-' * (length - 1)}:") # same size as humanize
       end
       $stdout.puts('|')
 
@@ -48,24 +55,33 @@ class BenchmarkDriver::Output::Markdown
   # @param [BenchmarkDriver::Job] job
   def with_job(job, &block)
     if @with_benchmark
-      $stdout.print("|%-#{@name_length}s  " % job.name)
+      @job_context_result = {} if @context_names.size > 1
+
+      $stdout.printf("|%-*s  ", @name_length, job.name)
     end
     block.call
   ensure
     if @with_benchmark
       $stdout.puts('|')
+      compare_executables if @compare && @context_names.size > 1
     end
   end
 
   # @param [BenchmarkDriver::Context] context
   def with_context(context, &block)
+    @context = context
     block.call
   end
 
   # @param [BenchmarkDriver::Result] result
   def report(result)
+    if defined?(@job_context_result)
+      @job_context_result[@context] = result
+    end
+
     if @with_benchmark
-      $stdout.print("|%#{NAME_LENGTH}s" % humanize(result.values.fetch(@metrics.first)))
+      length = [NAME_LENGTH, @context.name.length].max
+      $stdout.printf("|%*s", length, humanize(result.values.fetch(@metrics.first)))
     else
       $stdout.print '.'
     end
@@ -83,15 +99,15 @@ class BenchmarkDriver::Output::Markdown
 
   def humanize(value)
     if BenchmarkDriver::Result::ERROR.equal?(value)
-      return "%#{NAME_LENGTH}s" % 'ERROR'
+      return sprintf("%*s", NAME_LENGTH, 'ERROR')
     elsif value == 0.0
-      return "%#{NAME_LENGTH}.3f" % 0.0
+      return sprintf("%*.3f", NAME_LENGTH, 0.0)
     elsif value < 0
       raise ArgumentError.new("Negative value: #{value.inspect}")
     end
 
     scale = (Math.log10(value) / 3).to_i
-    prefix = "%#{NAME_LENGTH - 1}.3f" % (value.to_f / (1000 ** scale))
+    prefix = sprintf("%*.3f", NAME_LENGTH - 1, (value.to_f / (1000 ** scale)))
     suffix =
       case scale
       when 1; 'k'
@@ -104,4 +120,30 @@ class BenchmarkDriver::Output::Markdown
       end
     "#{prefix}#{suffix}"
   end
+
+  def compare_executables
+    order = @metrics.first.larger_better ? :min_by : :max_by
+    worst, worst_result = @job_context_result.__send__(order) do |_, result|
+      result.values.first[1]
+    end
+    worst_result = worst_result.values.first[1]
+    $stdout.print("|", " " * (@name_length + 2))
+    @job_context_result.each do |context, result|
+      if context == worst
+        result = '-'
+      else
+        result = result.values.first[1]
+        if order == :min_by
+          result = result.fdiv(worst_result)
+        else
+          result = best_result.fdiv(worst_result)
+        end
+        result = sprintf("%.2fx", result)
+      end
+      length = [context.name.length, NAME_LENGTH].max
+      $stdout.printf("|%*s", length, result)
+    end
+    $stdout.puts('|')
+  end
+
 end

data/lib/benchmark_driver/repeater.rb

@@ -54,7 +54,7 @@ module BenchmarkDriver
           [values.map { |v| v[0] }.inject(&:+) / values.size.to_f, *rest]
         when :average
           values.first.size.times.map do |index|
-            values.map { |v| v[index] }.inject(&:+) / values.first.size.to_f
+            values.map { |v| v[index] }.inject(&:+) / values.size.to_f
           end
         else
           raise "unexpected rest_on_average #{rest_on_average.inspect}"

data/lib/benchmark_driver/runner/ips.rb

@@ -107,8 +107,8 @@ class BenchmarkDriver::Runner::Ips
 
     duration = Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       with_script(benchmark.render(result: f.path)) do |path|
-        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
-        if $?.success? && ((value = Float(f.read)) > 0)
+        success = execute(*context.executable.command, path, exception: false)
+        if success && ((value = Float(f.read)) > 0)
           value
         else
           BenchmarkDriver::Result::ERROR
@@ -137,10 +137,7 @@ class BenchmarkDriver::Runner::Ips
   end
 
   def with_script(script)
-    if @config.verbose >= 2
-      sep = '-' * 30
-      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
-    end
+    debug_output('Script', script) if @config.verbose >= 2
 
     Tempfile.open(['benchmark_driver-', '.rb']) do |f|
       f.puts script
@@ -149,11 +146,20 @@ class BenchmarkDriver::Runner::Ips
     end
   end
 
-  def execute(*args)
-    IO.popen(args, &:read) # TODO: print stdout if verbose=2
-    unless $?.success?
-      raise "Failed to execute: #{args.shelljoin} (status: #{$?.exitstatus})"
+  def execute(*args, exception: true)
+    $stderr.puts "$ #{args.shelljoin}" if @config.verbose >= 2
+
+    stdout = IO.popen(args, &:read)
+    debug_output('Command output', stdout) if @config.verbose >= 2
+    if exception && !$?.success?
+      raise "Failed to execute: #{args.shelljoin} (status: #{$?})"
     end
+    $?.success?
+  end
+
+  def debug_output(name, text)
+    sep = '-' * 30
+    $stdout.puts "\n\n#{sep}[#{name} begin]#{sep}\n#{text}#{sep}[#{name} end]#{sep}\n\n"
   end
 
   WarmupScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count, :first_warmup_duration, :second_warmup_duration) do

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.15.10'
+  VERSION = '0.15.15'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.15.10
+  version: 0.15.15
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-01 00:00:00.000000000 Z
+date: 2020-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.0.pre1
 signing_key: 
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby