benchmark_driver 0.14.13 → 0.14.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22df737d5e378a5ea29b9667ffc96fc767427f27b0bb9e21b548b4fb5caf4463
-  data.tar.gz: fe45db762725218c739bb412a06189221fb6f5cb0e89a27071a1d302ce90c5bd
+  metadata.gz: 576179b82e3d7f201f8a58373e279219b2a20ffc7a74bb6133b09655db54ec45
+  data.tar.gz: c57f3b432fd5f7309f6bba0e318b75e425bb8e1e17e70916eed60b8e7a3b7785
 SHA512:
-  metadata.gz: 5d903d9b8d9b821f6eefbd21defc201714f6ec2e5436ca9374228756873a7693f37c4974cff38bfb2da01711bf394b8c516c8f6ce974ad19574a484ed780d4a7
-  data.tar.gz: 9b7e96da26336be348c4660328330c467f4f92ded7ad60538c23a6bfeb72fb58eddc8e7b7f1cedcccd30d340f34869191b71466e093c2ff93e2b6517ec658c87
+  metadata.gz: df832e8687b266851d5339039fa2e3a2b53d49ce8a25e5f2d7cf2c7691bd7f8c8394148c1d3de4c8dd0c7b5ab6d97f7a8e0ea923914478b7afc358c3d731178a
+  data.tar.gz: 569980b49fbcaf4515698f0519d7aa025bfe9d4acf7281d7d34cd85324e5afd0622b194417d744da1526b6ab2d7b5a15cc89c69d4cef9154650822231ab1df48

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v0.14.14
+
+- Add `all` output plugin to compare all sorted results
+
 # v0.14.13
 
 - Add macOS support to memory runner
@@ -87,7 +91,7 @@
 
 - Add support for benchmark with different versions of gems and preludes
   - Only YAML is supported for now, Ruby interface will come later
-- [breaking change] Runner plugin interface is sightly changed
+- [breaking change] Runner plugin interface is slightly changed
 
 # v0.12.0
 

data/README.md

@@ -272,7 +272,7 @@ ips, time, memory, once
 |:-------|:------------|
 | ips | Iteration per second (default) |
 | time | Elapsed seconds |
-| memory | Max resident set. This is supported only on Linux for now. |
+| memory | Max resident set. This is supported only on Linux and macOS for now. |
 | once | Forces `loop_count` to 1 for testing |
 | ruby\_stdout | Special runner to integrate existing benchmarks |
 
@@ -360,7 +360,7 @@ See following example:
 
 * https://github.com/benchmark-driver/optcarrot
 
-If you benchmark can run with `ruby foo bar`, specify `foo bar` to `command:`.
+If your benchmark can run with `ruby foo bar`, specify `foo bar` to `command:`.
 Then write `stdout_to_metrics:` to convert stdout to metrics. This runner can be used only with YAML interface for now.
 
 ```

data/lib/benchmark_driver/metric.rb

@@ -30,6 +30,7 @@ module BenchmarkDriver
   # Everything that can be known after running benchmark
   Result = ::BenchmarkDriver::Struct.new(
     :values,      # @param [Hash{ BenchmarkDriver::Metric => Float }] - Main benchmark results
+    :all_values,  # @param [Hash{ BenchmarkDriver::Metric => Float }] - All benchmark results. Used by --output=all (optional)
     :duration,    # @param [Float,nil] - Time taken to run the benchmark job (optional)
     :loop_count,  # @param [Integer,nil] - Times to run the benchmark job (optional)
     :environment, # @param [Hash] - Any other key -> value pairs to express the benchmark context

data/lib/benchmark_driver/output.rb

@@ -73,11 +73,13 @@ module BenchmarkDriver
       end
     end
 
-    # @param [Float] value
+    # @param [Hash{ BenchmarkDriver::Metric => Float }] values
+    # @param [Hash{ BenchmarkDriver::Metric => [Float] },nil] values
     # @param [BenchmarkDriver::Metric] metic
-    def report(values:, duration: nil, loop_count: nil, environment: {})
+    def report(values:, all_values: nil, duration: nil, loop_count: nil, environment: {})
       result = BenchmarkDriver::Result.new(
         values: values,
+        all_values: all_values,
         duration: duration,
         loop_count: loop_count,
         environment: environment,

data/lib/benchmark_driver/output/all.rb

@@ -0,0 +1,104 @@
+class BenchmarkDriver::Output::All
+  NAME_LENGTH = 20
+  CONTEXT_LENGTH = 20
+
+  # @param [Array<BenchmarkDriver::Metric>] metrics
+  # @param [Array<BenchmarkDriver::Job>] jobs
+  # @param [Array<BenchmarkDriver::Context>] contexts
+  def initialize(metrics:, jobs:, contexts:)
+    @metrics = metrics
+    @job_names = jobs.map(&:name)
+    @context_names = contexts.map(&:name)
+    @name_length = [@job_names.map(&:length).max, NAME_LENGTH].max
+  end
+
+  def with_warmup(&block)
+    without_stdout_buffering do
+      $stdout.puts 'Warming up --------------------------------------'
+      # TODO: show exec name if it has multiple ones
+      block.call
+    end
+  end
+
+  def with_benchmark(&block)
+    @job_context_result = Hash.new do |hash, job|
+      hash[job] = {}
+    end
+
+    result = without_stdout_buffering do
+      $stdout.puts 'Calculating -------------------------------------'
+      if @context_names.size > 1
+        $stdout.print(' ' * @name_length)
+        @context_names.each do |context_name|
+          $stdout.print(" %#{CONTEXT_LENGTH}s " % context_name)
+        end
+        $stdout.puts
+      end
+
+      block.call
+    end
+    result
+  end
+
+  # @param [BenchmarkDriver::Job] job
+  def with_job(job, &block)
+    @job_name = "%#{@name_length}s" % job.name
+    $stdout.print(@job_name)
+    @context_values = {}
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Context] context
+  def with_context(context, &block)
+    @context = context
+    block.call
+  end
+
+  # @param [BenchmarkDriver::Result] result
+  def report(result)
+    if result.all_values.nil? || !defined?(@context_values)
+      $stdout.puts(" %#{[CONTEXT_LENGTH, @context.name.length].max}s " % result.values.values.first.to_s)
+      return
+    end
+
+    num_values = result.all_values.values.first.size
+    if @context_values.empty?
+      print("\r")
+    else
+      print("\e[#{num_values}F")
+    end
+    @context_values[@context] = result.all_values.values.first.sort
+
+    precision = result.values.values.first.to_s.sub(/\A\d+\./, '').length
+    num_values.times do |i|
+      if i == 0
+        $stdout.print(@job_name)
+      else
+        print(" " * [@job_name.length, NAME_LENGTH].max)
+      end
+
+      @context_values.each do |context, values|
+        $stdout.print(" %#{[CONTEXT_LENGTH, context.name.length].max}.#{precision}f " % values[i])
+      end
+      (@context_names - @context_values.keys.map(&:name)).each do |context_name|
+        print(" " * ([CONTEXT_LENGTH, context_name.length].max + 2))
+      end
+
+      if i == 0
+        $stdout.puts(@metrics.first.unit)
+      else
+        $stdout.puts
+      end
+    end
+  end
+
+  private
+
+  # benchmark_driver ouputs logs ASAP. This enables sync flag for it.
+  def without_stdout_buffering
+    sync, $stdout.sync = $stdout.sync, true
+    yield
+  ensure
+    $stdout.sync = sync
+  end
+end

data/lib/benchmark_driver/repeater.rb

@@ -1,25 +1,36 @@
+require 'benchmark_driver/struct'
+
 module BenchmarkDriver
   # Repeat calling block and return desired result: "best", "worst" or "average".
   module Repeater
     VALID_TYPES = %w[best worst average]
 
+    RepeatResult = ::BenchmarkDriver::Struct.new(
+      :value, # the value desired by --repeat-result
+      :all_values, # all benchmark results. Used by --output=all.
+    )
+
     class << self
       # `block.call` can return multiple objects, but the first one is used for sort.
       # When `config.repeat_result == 'average'`, how to deal with rest objects is decided
       # by `:rest_on_average` option.
+      #
+      # @param [Proc] block - returns Float or [Float, ...]
       def with_repeat(config:, larger_better:, rest_on_average: :first, &block)
         values = config.repeat_count.times.map { block.call }
 
-        case config.repeat_result
-        when 'best'
-          best_result(values, larger_better)
-        when 'worst'
-          best_result(values, !larger_better)
-        when 'average'
-          average_result(values, rest_on_average)
-        else
-          raise "unexpected repeat_result #{config.repeat_result.inspect}"
-        end
+        desired_value =
+          case config.repeat_result
+          when 'best'
+            best_result(values, larger_better)
+          when 'worst'
+            best_result(values, !larger_better)
+          when 'average'
+            average_result(values, rest_on_average)
+          else
+            raise "unexpected repeat_result #{config.repeat_result.inspect}"
+          end
+        RepeatResult.new(value: desired_value, all_values: extract_values(values))
       end
 
       private
@@ -47,6 +58,12 @@ module BenchmarkDriver
           raise "unexpected rest_on_average #{rest_on_average.inspect}"
         end
       end
+
+      def extract_values(values)
+        values.map do |value|
+          value.is_a?(Array) ? value.first : value
+        end
+      end
     end
   end
 end

data/lib/benchmark_driver/runner/command_stdout.rb

@@ -64,7 +64,7 @@ class BenchmarkDriver::Runner::CommandStdout
         @output.with_job(name: job.name) do
           @contexts.each do |context|
             exec = context.executable
-            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: metric.larger_better) do
+            result = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: metric.larger_better) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
               end
@@ -75,7 +75,7 @@ class BenchmarkDriver::Runner::CommandStdout
             end
 
             @output.with_context(name: exec.name, executable: exec) do
-              @output.report(values: { metric => value })
+              @output.report(values: { metric => result.value }, all_values: { metric => result.all_values })
             end
           end
         end

data/lib/benchmark_driver/runner/ips.rb

@@ -52,11 +52,17 @@ class BenchmarkDriver::Runner::Ips
         @output.with_job(name: job.name) do
           job.runnable_contexts(@contexts).each do |context|
             repeat_params = { config: @config, larger_better: true, rest_on_average: :average }
-            value, duration = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
+            result = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               run_benchmark(job, context: context)
             end
+            value, duration = result.value
             @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
-              @output.report(values: { metric => value }, duration: duration, loop_count: job.loop_count)
+              @output.report(
+                values: { metric => value },
+                all_values: { metric => result.all_values },
+                duration: duration,
+                loop_count: job.loop_count,
+              )
             end
           end
         end

data/lib/benchmark_driver/runner/memory.rb

@@ -48,11 +48,11 @@ class BenchmarkDriver::Runner::Memory
       jobs.each do |job|
         @output.with_job(name: job.name) do
           job.runnable_contexts(@contexts).each do |context|
-            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+            result = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
               run_benchmark(job, context: context)
             end
             @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
-              @output.report(values: { METRIC => value }, loop_count: job.loop_count)
+              @output.report(values: { METRIC => result.value }, all_values: { METRIC => result.all_values }, loop_count: job.loop_count)
             end
           end
         end

data/lib/benchmark_driver/runner/ruby_stdout.rb

@@ -82,7 +82,7 @@ class BenchmarkDriver::Runner::RubyStdout
           @contexts.each do |context|
             exec = context.executable
             repeat_params = { config: @config, larger_better: metric.larger_better }
-            value, environment = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
+            result = BenchmarkDriver::Repeater.with_repeat(repeat_params) do
               stdout = with_chdir(job.working_directory) do
                 with_ruby_prefix(exec) { execute(*exec.command, *job.command) }
               end
@@ -93,9 +93,14 @@ class BenchmarkDriver::Runner::RubyStdout
               )
               [script.value, script.environment]
             end
+            value, environment = result.value
 
             @output.with_context(name: exec.name, executable: exec) do
-              @output.report(values: { metric => value }, environment: environment)
+              @output.report(
+                values: { metric => value },
+                all_values: { metric => result.all_values },
+                environment: environment,
+              )
             end
           end
         end

data/lib/benchmark_driver/version.rb

@@ -1,3 +1,3 @@
 module BenchmarkDriver
-  VERSION = '0.14.13'
+  VERSION = '0.14.14'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: benchmark_driver
 version: !ruby/object:Gem::Version
-  version: 0.14.13
+  version: 0.14.14
 platform: ruby
 authors:
 - Takashi Kokubun
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-24 00:00:00.000000000 Z
+date: 2019-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,6 +106,7 @@ files:
 - lib/benchmark_driver/job_parser.rb
 - lib/benchmark_driver/metric.rb
 - lib/benchmark_driver/output.rb
+- lib/benchmark_driver/output/all.rb
 - lib/benchmark_driver/output/compare.rb
 - lib/benchmark_driver/output/markdown.rb
 - lib/benchmark_driver/output/record.rb
@@ -142,8 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fully-featured accurate benchmark driver for Ruby